From f8814a3910f0c62215a264c67026d81a422b54aa Mon Sep 17 00:00:00 2001 From: "dandan.sha" <20832776@qunar.com> Date: Fri, 7 Dec 2018 13:56:42 +0800 Subject: [PATCH] init --- .gitignore | 37 ++ Makefile | 2 + README.md | 43 ++ docs/cn/arch.md | 67 +++ docs/cn/code.md | 36 ++ docs/cn/consumer.md | 122 +++++ docs/cn/contributing.md | 5 + docs/cn/delay.md | 46 ++ docs/cn/design.md | 38 ++ docs/cn/ha.md | 21 + docs/cn/install.md | 159 ++++++ docs/cn/monitor.md | 55 ++ docs/cn/opensource.md | 10 + docs/cn/producer.md | 107 ++++ docs/cn/quickstart.md | 30 ++ docs/cn/support.md | 10 + docs/cn/tag.md | 30 ++ docs/cn/trace.md | 27 + docs/images/arch1.png | Bin 0 -> 27728 bytes docs/images/arch3.png | Bin 0 -> 26945 bytes docs/images/arch4.png | Bin 0 -> 39566 bytes docs/images/design1.png | Bin 0 -> 35285 bytes docs/images/design2.png | Bin 0 -> 23696 bytes docs/images/design3.png | Bin 0 -> 24493 bytes docs/images/design4.png | Bin 0 -> 27189 bytes docs/images/ha.png | Bin 0 -> 20525 bytes docs/images/support1.png | Bin 0 -> 15482 bytes pom.xml | 292 ++++++++++ qmq-api/pom.xml | 19 + .../main/java/qunar/tc/qmq/BaseConsumer.java | 27 + .../src/main/java/qunar/tc/qmq/Filter.java | 45 ++ .../java/qunar/tc/qmq/FilterAttachable.java | 29 + .../qunar/tc/qmq/IdempotentAttachable.java | 27 + .../java/qunar/tc/qmq/IdempotentChecker.java | 51 ++ .../java/qunar/tc/qmq/ListenerHolder.java | 26 + .../src/main/java/qunar/tc/qmq/Message.java | 173 ++++++ .../java/qunar/tc/qmq/MessageConsumer.java | 52 ++ .../java/qunar/tc/qmq/MessageListener.java | 26 + .../java/qunar/tc/qmq/MessageProducer.java | 47 ++ .../tc/qmq/MessageSendStateListener.java | 40 ++ .../java/qunar/tc/qmq/NeedRetryException.java | 54 ++ .../java/qunar/tc/qmq/ProduceMessage.java | 42 ++ .../main/java/qunar/tc/qmq/PullConsumer.java | 100 ++++ .../java/qunar/tc/qmq/SubscribeParam.java | 94 ++++ .../src/main/java/qunar/tc/qmq/TagType.java | 54 ++ qmq-client/pom.xml | 60 +++ .../tc/qmq/broker/BrokerClusterInfo.java | 57 ++ .../qunar/tc/qmq/broker/BrokerGroupInfo.java | 99 ++++ .../tc/qmq/broker/BrokerLoadBalance.java | 24 + .../qunar/tc/qmq/broker/BrokerService.java | 35 ++ .../qunar/tc/qmq/broker/CircuitBreaker.java | 153 ++++++ .../tc/qmq/broker/impl/BrokerServiceImpl.java | 153 ++++++ .../broker/impl/PollBrokerLoadBalance.java | 79 +++ .../qunar/tc/qmq/common/AtomicConfig.java | 80 +++ .../tc/qmq/common/AtomicIntegerConfig.java | 65 +++ .../qunar/tc/qmq/common/ClientIdProvider.java | 25 + .../qmq/common/ClientIdProviderFactory.java | 27 + .../qmq/common/DefaultClientIdProvider.java | 64 +++ .../qunar/tc/qmq/common/MapKeyBuilder.java | 48 ++ .../qunar/tc/qmq/common/StatusSource.java | 37 ++ .../qunar/tc/qmq/common/SwitchWaiter.java | 90 ++++ .../java/qunar/tc/qmq/common/TimerUtil.java | 35 ++ .../qmq/config/NettyClientConfigManager.java | 39 ++ .../tc/qmq/config/PullSubjectsConfig.java | 158 ++++++ .../tc/qmq/consumer/BaseMessageHandler.java | 169 ++++++ .../qunar/tc/qmq/consumer/ConsumeMessage.java | 84 +++ .../qmq/consumer/MessageConsumerProvider.java | 148 ++++++ .../annotation/ConsumerAnnotationScanner.java | 204 +++++++ .../tc/qmq/consumer/annotation/EnableQmq.java | 36 ++ .../annotation/GeneratedListener.java | 60 +++ .../QmqClientBeanDefinitionParser.java | 57 ++ .../annotation/QmqClientNamespaceHandler.java | 31 ++ .../qmq/consumer/annotation/QmqConsumer.java | 85 +++ .../annotation/QmqConsumerRegister.java | 63 +++ .../CreatePullConsumerException.java | 38 ++ .../exception/DuplicateListenerException.java | 36 ++ .../handler/IdempotentCheckerFilter.java | 55 ++ .../consumer/handler/MessageDistributor.java | 67 +++ .../tc/qmq/consumer/handler/QTraceFilter.java | 67 +++ .../idempotent/AbstractIdempotentChecker.java | 142 +++++ .../idempotent/JdbcIdempotentChecker.java | 77 +++ .../TransactionalJdbcIdempotentChecker.java | 72 +++ .../consumer/pull/AbstractPullConsumer.java | 126 +++++ .../qmq/consumer/pull/AbstractPullEntry.java | 188 +++++++ .../qunar/tc/qmq/consumer/pull/AckEntry.java | 114 ++++ .../qunar/tc/qmq/consumer/pull/AckHelper.java | 72 +++ .../qunar/tc/qmq/consumer/pull/AckHook.java | 24 + .../tc/qmq/consumer/pull/AckSendEntry.java | 59 +++ .../tc/qmq/consumer/pull/AckSendInfo.java | 55 ++ .../tc/qmq/consumer/pull/AckSendQueue.java | 387 ++++++++++++++ .../tc/qmq/consumer/pull/AckService.java | 269 ++++++++++ .../qmq/consumer/pull/AlwaysPullStrategy.java | 29 + .../tc/qmq/consumer/pull/ConsumeParam.java | 107 ++++ .../consumer/pull/DefaultPullConsumer.java | 192 +++++++ .../consumer/pull/DelayMessageService.java | 122 +++++ .../tc/qmq/consumer/pull/PlainPullEntry.java | 61 +++ .../consumer/pull/PullConsumerFactory.java | 72 +++ .../qunar/tc/qmq/consumer/pull/PullEntry.java | 222 ++++++++ .../qmq/consumer/pull/PullMessageFuture.java | 118 +++++ .../qunar/tc/qmq/consumer/pull/PullParam.java | 179 +++++++ .../tc/qmq/consumer/pull/PullRegister.java | 220 ++++++++ .../tc/qmq/consumer/pull/PullResult.java | 49 ++ .../tc/qmq/consumer/pull/PullService.java | 263 +++++++++ .../tc/qmq/consumer/pull/PullStrategy.java | 23 + .../tc/qmq/consumer/pull/PulledMessage.java | 57 ++ .../consumer/pull/PulledMessageFilter.java | 28 + .../tc/qmq/consumer/pull/PushConsumer.java | 33 ++ .../qmq/consumer/pull/PushConsumerImpl.java | 153 ++++++ .../tc/qmq/consumer/pull/SendMessageBack.java | 36 ++ .../consumer/pull/SendMessageBackImpl.java | 197 +++++++ .../qmq/consumer/pull/WeightLoadBalance.java | 110 ++++ .../qmq/consumer/pull/WeightPullStrategy.java | 44 ++ .../consumer/pull/exception/AckException.java | 38 ++ .../pull/exception/PullException.java | 38 ++ .../exception/SendMessageBackException.java | 38 ++ .../consumer/register/ConsumerRegister.java | 33 ++ .../tc/qmq/consumer/register/RegistParam.java | 75 +++ .../ConsumerStateChangedListener.java | 27 + .../qunar/tc/qmq/metainfoclient/MetaInfo.java | 56 ++ .../tc/qmq/metainfoclient/MetaInfoClient.java | 36 ++ .../metainfoclient/MetaInfoClientHandler.java | 106 ++++ .../MetaInfoClientNettyImpl.java | 153 ++++++ .../qmq/metainfoclient/MetaInfoService.java | 298 +++++++++++ .../qunar/tc/qmq/producer/ConfigCenter.java | 82 +++ .../qmq/producer/MessageProducerProvider.java | 183 +++++++ .../tc/qmq/producer/ProduceMessageImpl.java | 270 ++++++++++ .../qunar/tc/qmq/producer/QueueSender.java | 34 ++ .../tc/qmq/producer/RegistryResolver.java | 34 ++ .../tc/qmq/producer/SendErrorHandler.java | 34 ++ .../qmq/producer/idgenerator/IdGenerator.java | 26 + .../TimestampAndHostIdGenerator.java | 52 ++ .../sender/AbstractRouterManager.java | 79 +++ .../tc/qmq/producer/sender/Connection.java | 38 ++ .../producer/sender/MessageSenderGroup.java | 88 ++++ .../qmq/producer/sender/NettyConnection.java | 196 +++++++ .../producer/sender/NettyProducerClient.java | 56 ++ .../tc/qmq/producer/sender/NettyRouter.java | 58 ++ .../producer/sender/NettyRouterManager.java | 90 ++++ .../tc/qmq/producer/sender/NopRoute.java | 58 ++ .../qmq/producer/sender/RPCQueueSender.java | 146 +++++ .../qunar/tc/qmq/producer/sender/Route.java | 25 + .../qunar/tc/qmq/producer/sender/Router.java | 24 + .../tc/qmq/producer/sender/RouterManager.java | 37 ++ .../qmq/tracing/QmqMessageExtractAdapter.java | 50 ++ .../qmq/tracing/QmqMessageInjectAdapter.java | 42 ++ .../java/qunar/tc/qmq/tracing/TraceUtil.java | 73 +++ .../src/main/resources/META-INF/qmq-2.0.0.xsd | 33 ++ .../main/resources/META-INF/spring.handlers | 1 + .../main/resources/META-INF/spring.schemas | 2 + qmq-common/pom.xml | 49 ++ .../java/qunar/tc/qmq/base/BaseMessage.java | 399 ++++++++++++++ .../qunar/tc/qmq/base/ClientRequestType.java | 36 ++ .../qunar/tc/qmq/base/LargeStringUtil.java | 64 +++ .../java/qunar/tc/qmq/base/MessageHeader.java | 90 ++++ .../qunar/tc/qmq/base/OnOfflineState.java | 44 ++ .../java/qunar/tc/qmq/base/RawMessage.java | 54 ++ .../qunar/tc/qmq/batch/BatchExecutor.java | 113 ++++ .../java/qunar/tc/qmq/batch/Processor.java | 28 + .../java/qunar/tc/qmq/common/ClientType.java | 65 +++ .../java/qunar/tc/qmq/common/Disposable.java | 26 + .../tc/qmq/concurrent/NamedThreadFactory.java | 49 ++ .../tc/qmq/configuration/DynamicConfig.java | 48 ++ .../configuration/DynamicConfigFactory.java | 25 + .../configuration/DynamicConfigLoader.java | 50 ++ .../qunar/tc/qmq/configuration/Listener.java | 25 + .../configuration/local/ConfigWatcher.java | 101 ++++ .../local/LocalDynamicConfig.java | 238 +++++++++ .../local/LocalDynamicConfigFactory.java | 51 ++ .../java/qunar/tc/qmq/meta/BrokerCluster.java | 39 ++ .../java/qunar/tc/qmq/meta/BrokerGroup.java | 116 ++++ .../qunar/tc/qmq/meta/BrokerGroupKind.java | 45 ++ .../java/qunar/tc/qmq/meta/BrokerState.java | 52 ++ .../qunar/tc/qmq/meta/MetaServerLocator.java | 101 ++++ .../java/qunar/tc/qmq/metrics/Metrics.java | 73 +++ .../tc/qmq/metrics/MetricsConstants.java | 23 + .../qunar/tc/qmq/metrics/MockRegistry.java | 99 ++++ .../java/qunar/tc/qmq/metrics/QmqCounter.java | 31 ++ .../java/qunar/tc/qmq/metrics/QmqMeter.java | 27 + .../tc/qmq/metrics/QmqMetricRegistry.java | 35 ++ .../java/qunar/tc/qmq/metrics/QmqTimer.java | 27 + .../exceptions/BlockMessageException.java | 30 ++ .../exceptions/DuplicateMessageException.java | 31 ++ .../service/exceptions/MessageException.java | 66 +++ .../java/qunar/tc/qmq/utils/CharsetUtils.java | 47 ++ .../java/qunar/tc/qmq/utils/Checksums.java | 40 ++ .../main/java/qunar/tc/qmq/utils/Crc32.java | 399 ++++++++++++++ .../java/qunar/tc/qmq/utils/DelayUtil.java | 34 ++ .../main/java/qunar/tc/qmq/utils/Flags.java | 39 ++ .../java/qunar/tc/qmq/utils/ListUtils.java | 54 ++ .../java/qunar/tc/qmq/utils/NetworkUtils.java | 97 ++++ .../java/qunar/tc/qmq/utils/ObjectUtils.java | 31 ++ .../tc/qmq/utils/PayloadHolderUtils.java | 101 ++++ .../main/java/qunar/tc/qmq/utils/PidUtil.java | 32 ++ .../qunar/tc/qmq/utils/RetrySubjectUtils.java | 91 ++++ ....tc.qmq.configuration.DynamicConfigFactory | 1 + qmq-delay-server/pom.xml | 39 ++ .../tc/qmq/delay/DefaultDelayLogFacade.java | 198 +++++++ .../qunar/tc/qmq/delay/DelayLogFacade.java | 89 ++++ .../qunar/tc/qmq/delay/EventListener.java | 27 + .../java/qunar/tc/qmq/delay/LogFlusher.java | 53 ++ .../delay/MessageIterateEventListener.java | 65 +++ .../tc/qmq/delay/MessageLogReplayer.java | 164 ++++++ .../qunar/tc/qmq/delay/ScheduleIndex.java | 60 +++ .../java/qunar/tc/qmq/delay/Switchable.java | 29 + .../tc/qmq/delay/base/AppendException.java | 28 + .../tc/qmq/delay/base/GroupSendException.java | 27 + .../qunar/tc/qmq/delay/base/LongHashSet.java | 140 +++++ .../qmq/delay/base/ReceivedDelayMessage.java | 81 +++ .../tc/qmq/delay/base/ReceivedResult.java | 55 ++ .../qmq/delay/base/SegmentBufferExtend.java | 39 ++ .../tc/qmq/delay/cleaner/LogCleaner.java | 96 ++++ .../config/DefaultStoreConfiguration.java | 145 +++++ .../qmq/delay/config/StoreConfiguration.java | 57 ++ .../tc/qmq/delay/container/Bootstrap.java | 26 + .../tc/qmq/delay/meta/BrokerRoleManager.java | 35 ++ .../java/qunar/tc/qmq/delay/monitor/QMon.java | 124 +++++ .../qunar/tc/qmq/delay/receiver/Invoker.java | 27 + .../ReceivedDelayMessageProcessor.java | 212 ++++++++ .../qunar/tc/qmq/delay/receiver/Receiver.java | 270 ++++++++++ .../tc/qmq/delay/receiver/filter/Filter.java | 28 + .../receiver/filter/OverDelayException.java | 28 + .../receiver/filter/OverDelayFilter.java | 41 ++ .../receiver/filter/PastDelayFilter.java | 36 ++ .../receiver/filter/ReceiveFilterChain.java | 54 ++ .../delay/receiver/filter/ValidateFilter.java | 38 ++ .../tc/qmq/delay/sender/DelayProcessor.java | 30 ++ .../tc/qmq/delay/sender/NettySender.java | 60 +++ .../qunar/tc/qmq/delay/sender/Sender.java | 34 ++ .../tc/qmq/delay/sender/SenderExecutor.java | 120 +++++ .../tc/qmq/delay/sender/SenderGroup.java | 240 +++++++++ .../tc/qmq/delay/sender/SenderProcessor.java | 173 ++++++ .../tc/qmq/delay/startup/ServerWrapper.java | 186 +++++++ .../store/DefaultDelaySegmentValidator.java | 33 ++ .../delay/store/DelaySegmentValidator.java | 30 ++ .../qmq/delay/store/IterateOffsetManager.java | 97 ++++ .../store/ScheduleLogValidatorSupport.java | 139 +++++ .../tc/qmq/delay/store/VisitorAccessor.java | 33 ++ .../store/appender/DispatchLogAppender.java | 52 ++ .../qmq/delay/store/appender/LogAppender.java | 34 ++ .../store/appender/ScheduleSetAppender.java | 84 +++ .../qmq/delay/store/log/AbstractDelayLog.java | 63 +++ .../delay/store/log/AbstractDelaySegment.java | 169 ++++++ .../log/AbstractDelaySegmentContainer.java | 134 +++++ .../tc/qmq/delay/store/log/DelaySegment.java | 48 ++ .../tc/qmq/delay/store/log/DispatchLog.java | 91 ++++ .../delay/store/log/DispatchLogSegment.java | 108 ++++ .../log/DispatchLogSegmentContainer.java | 183 +++++++ .../qunar/tc/qmq/delay/store/log/Log.java | 31 ++ .../tc/qmq/delay/store/log/MessageLog.java | 165 ++++++ .../store/log/MessageSegmentContainer.java | 407 ++++++++++++++ .../tc/qmq/delay/store/log/ScheduleLog.java | 171 ++++++ .../store/log/ScheduleOffsetResolver.java | 39 ++ .../tc/qmq/delay/store/log/ScheduleSet.java | 64 +++ .../delay/store/log/ScheduleSetSegment.java | 106 ++++ .../log/ScheduleSetSegmentContainer.java | 147 ++++++ .../qmq/delay/store/log/SegmentContainer.java | 31 ++ .../model/AppendDispatchRecordResult.java | 45 ++ .../delay/store/model/AppendLogResult.java | 50 ++ .../model/AppendMessageRecordResult.java | 45 ++ .../delay/store/model/AppendRecordResult.java | 62 +++ .../model/AppendScheduleLogRecordResult.java | 47 ++ .../delay/store/model/DispatchLogRecord.java | 71 +++ .../tc/qmq/delay/store/model/LogRecord.java | 42 ++ .../delay/store/model/LogRecordHeader.java | 63 +++ .../delay/store/model/MessageLogAttrEnum.java | 48 ++ .../delay/store/model/MessageLogRecord.java | 81 +++ .../delay/store/model/NopeRecordResult.java | 45 ++ .../delay/store/model/RawMessageExtend.java | 49 ++ .../qmq/delay/store/model/RecordResult.java | 31 ++ .../delay/store/model/ScheduleSetRecord.java | 88 ++++ .../store/model/ScheduleSetSequence.java | 39 ++ .../store/visitor/AbstractLogVisitor.java | 97 ++++ .../store/visitor/DelayMessageLogVisitor.java | 232 ++++++++ .../store/visitor/DispatchLogVisitor.java | 45 ++ .../qmq/delay/store/visitor/LogVisitor.java | 33 ++ .../store/visitor/ScheduleIndexVisitor.java | 95 ++++ .../sync/master/AbstractLogSyncWorker.java | 89 ++++ .../master/DelaySyncRequestProcessor.java | 188 +++++++ .../sync/master/DispatchLogSyncWorker.java | 187 +++++++ .../sync/master/HeartbeatSyncWorker.java | 81 +++ .../sync/master/MasterSyncNettyServer.java | 53 ++ .../sync/master/MessageLogSyncWorker.java | 123 +++++ .../sync/slave/DispatchLogSyncProcessor.java | 63 +++ .../delay/sync/slave/HeartBeatProcessor.java | 60 +++ .../sync/slave/MessageLogSyncProcessor.java | 61 +++ .../delay/sync/slave/SlaveSynchronizer.java | 162 ++++++ .../delay/sync/slave/SyncLogProcessor.java | 29 + .../tc/qmq/delay/wheel/HashedWheelTimer.java | 497 ++++++++++++++++++ .../tc/qmq/delay/wheel/WheelLoadCursor.java | 123 +++++ .../tc/qmq/delay/wheel/WheelTickManager.java | 309 +++++++++++ qmq-dist/assembly/bin.xml | 35 ++ qmq-dist/bin/base.sh | 20 + qmq-dist/bin/broker-env.sh | 5 + qmq-dist/bin/broker.sh | 67 +++ qmq-dist/bin/delay-env.sh | 5 + qmq-dist/bin/delay.sh | 67 +++ qmq-dist/bin/metaserver-env.sh | 5 + qmq-dist/bin/metaserver.sh | 67 +++ qmq-dist/bin/tools-env.sh | 6 + qmq-dist/bin/tools.sh | 19 + qmq-dist/conf/broker.properties | 4 + qmq-dist/conf/datasource.properties | 5 + qmq-dist/conf/delay.properties | 4 + qmq-dist/conf/logback.xml | 14 + qmq-dist/conf/metaserver.properties | 1 + qmq-dist/conf/valid-api-tokens.properties | 0 qmq-dist/pom.xml | 58 ++ qmq-metaserver/pom.xml | 100 ++++ qmq-metaserver/sql/init.sql | 96 ++++ .../tc/qmq/meta/cache/AliveClientManager.java | 151 ++++++ .../qmq/meta/cache/CachedMetaInfoManager.java | 231 ++++++++ .../meta/cache/CachedOfflineStateManager.java | 135 +++++ .../meta/container/WebContainerListener.java | 43 ++ .../meta/loadbalance/AbstractLoadBalance.java | 37 ++ .../tc/qmq/meta/loadbalance/LoadBalance.java | 27 + .../meta/loadbalance/RandomLoadBalance.java | 41 ++ .../tc/qmq/meta/management/ActionResult.java | 61 +++ .../qmq/meta/management/AddBrokerAction.java | 87 +++ .../meta/management/AddNewSubjectAction.java | 71 +++ .../AddSubjectBrokerGroupAction.java | 94 ++++ .../management/ExtendSubjectRouteAction.java | 90 ++++ .../management/ListBrokerGroupsAction.java | 38 ++ .../meta/management/ListBrokersAction.java | 55 ++ .../management/ListSubjectRoutesAction.java | 38 ++ .../meta/management/MetaManagementAction.java | 27 + .../MetaManagementActionSupplier.java | 44 ++ .../management/QuerySubjectRouteAction.java | 59 +++ .../RemoveSubjectBrokerGroupAction.java | 130 +++++ .../meta/management/ReplaceBrokerAction.java | 99 ++++ .../management/TokenVerificationAction.java | 59 +++ .../qunar/tc/qmq/meta/model/BrokerMeta.java | 65 +++ .../tc/qmq/meta/model/ClientMetaInfo.java | 102 ++++ .../tc/qmq/meta/model/ClientOfflineState.java | 71 +++ .../tc/qmq/meta/model/GroupedConsumer.java | 80 +++ .../model/ReadonlyBrokerGroupSetting.java | 47 ++ .../qunar/tc/qmq/meta/model/SubjectInfo.java | 59 +++ .../qunar/tc/qmq/meta/model/SubjectRoute.java | 62 +++ .../java/qunar/tc/qmq/meta/monitor/QMon.java | 53 ++ .../processor/BrokerAcquireMetaProcessor.java | 110 ++++ .../processor/BrokerRegisterProcessor.java | 168 ++++++ .../processor/ClientRegisterProcessor.java | 96 ++++ .../meta/processor/ClientRegisterWorker.java | 166 ++++++ .../meta/route/SubjectConsumerService.java | 29 + .../meta/route/SubjectRegisterService.java | 25 + .../tc/qmq/meta/route/SubjectRouter.java | 30 ++ .../meta/route/impl/DefaultSubjectRouter.java | 184 +++++++ .../tc/qmq/meta/route/impl/DelayRouter.java | 87 +++ .../impl/SubjectConsumerServiceImpl.java | 68 +++ .../qunar/tc/qmq/meta/startup/Bootstrap.java | 47 ++ .../tc/qmq/meta/startup/ServerWrapper.java | 116 ++++ .../qunar/tc/qmq/meta/store/BrokerStore.java | 40 ++ .../qmq/meta/store/ClientMetaInfoStore.java | 29 + .../tc/qmq/meta/store/ClientOfflineStore.java | 40 ++ .../tc/qmq/meta/store/JdbcTemplateHolder.java | 68 +++ .../java/qunar/tc/qmq/meta/store/Store.java | 59 +++ .../qmq/meta/store/impl/BrokerStoreImpl.java | 99 ++++ .../store/impl/ClientMetaInfoStoreImpl.java | 47 ++ .../store/impl/ClientOfflineStoreImpl.java | 110 ++++ .../tc/qmq/meta/store/impl/DatabaseStore.java | 194 +++++++ .../tc/qmq/meta/store/impl/Serializer.java | 57 ++ .../tc/qmq/meta/utils/ClientLogUtils.java | 61 +++ .../qunar/tc/qmq/meta/utils/SubjectUtils.java | 32 ++ .../qunar/tc/qmq/meta/web/JsonResult.java | 39 ++ .../qmq/meta/web/MetaManagementServlet.java | 59 +++ .../web/MetaServerAddressSupplierServlet.java | 49 ++ .../tc/qmq/meta/web/OnOfflineServlet.java | 90 ++++ .../qunar/tc/qmq/meta/web/ResultStatus.java | 26 + .../qmq/meta/web/SubjectConsumerServlet.java | 67 +++ qmq-metrics-prometheus/pom.xml | 25 + .../prometheus/PrometheusQmqCounter.java | 52 ++ .../prometheus/PrometheusQmqGauge.java | 81 +++ .../prometheus/PrometheusQmqMeter.java | 42 ++ .../PrometheusQmqMetricRegistry.java | 180 +++++++ .../prometheus/PrometheusQmqTimer.java | 39 ++ .../qunar.tc.qmq.metrics.QmqMetricRegistry | 1 + qmq-remoting/pom.xml | 40 ++ .../qunar/tc/qmq/netty/DecodeHandler.java | 96 ++++ .../qunar/tc/qmq/netty/EncodeHandler.java | 65 +++ .../qunar/tc/qmq/netty/NettyClientConfig.java | 77 +++ .../qmq/netty/client/AbstractNettyClient.java | 102 ++++ .../tc/qmq/netty/client/ChannelWrapper.java | 43 ++ .../netty/client/HttpResponseCallback.java | 26 + .../tc/qmq/netty/client/NettyClient.java | 159 ++++++ .../qmq/netty/client/NettyClientHandler.java | 163 ++++++ .../client/NettyConnectManageHandler.java | 214 ++++++++ .../qunar/tc/qmq/netty/client/Response.java | 28 + .../tc/qmq/netty/client/ResponseFuture.java | 126 +++++ .../exception/BrokerRejectException.java | 29 + .../netty/exception/ClientSendException.java | 63 +++ .../qmq/netty/exception/EncodeException.java | 31 ++ .../netty/exception/RemoteBusyException.java | 25 + .../qmq/netty/exception/RemoteException.java | 40 ++ .../RemoteResponseUnreadableException.java | 24 + .../exception/RemoteTimeoutException.java | 40 ++ .../SubjectNotAssignedException.java | 34 ++ .../qunar/tc/qmq/protocol/CommandCode.java | 59 +++ .../java/qunar/tc/qmq/protocol/Datagram.java | 65 +++ .../qmq/protocol/MessagesPayloadHolder.java | 135 +++++ .../qunar/tc/qmq/protocol/PayloadHolder.java | 27 + .../qunar/tc/qmq/protocol/QMQSerializer.java | 90 ++++ .../tc/qmq/protocol/RemotingCommand.java | 48 ++ .../tc/qmq/protocol/RemotingCommandType.java | 45 ++ .../qunar/tc/qmq/protocol/RemotingHeader.java | 121 +++++ .../tc/qmq/protocol/consumer/AckRequest.java | 78 +++ .../consumer/AckRequestPayloadHolder.java | 42 ++ .../protocol/consumer/MetaInfoRequest.java | 138 +++++ .../MetaInfoRequestPayloadHolder.java | 37 ++ .../protocol/consumer/MetaInfoResponse.java | 92 ++++ .../tc/qmq/protocol/consumer/PullRequest.java | 147 ++++++ .../consumer/PullRequestPayloadHolder.java | 51 ++ .../producer/MessageProducerCode.java | 32 ++ .../tc/qmq/protocol/producer/SendResult.java | 50 ++ .../java/qunar/tc/qmq/util/ChannelUtil.java | 52 ++ .../java/qunar/tc/qmq/util/RemoteHelper.java | 76 +++ .../qunar/tc/qmq/util/RemotingBuilder.java | 70 +++ qmq-server-common/pom.xml | 50 ++ .../tc/qmq/base/ActionLogOffsetRequest.java | 52 ++ .../tc/qmq/base/ActionLogOffsetResponse.java | 122 +++++ .../tc/qmq/base/ConsumeManageRequest.java | 60 +++ .../java/qunar/tc/qmq/base/ConsumerLag.java | 43 ++ .../tc/qmq/base/QueryConsumerLagRequest.java | 42 ++ .../java/qunar/tc/qmq/base/SyncRequest.java | 54 ++ .../qunar/tc/qmq/concurrent/ActorSystem.java | 452 ++++++++++++++++ .../tc/qmq/configuration/BrokerConfig.java | 86 +++ .../tc/qmq/constants/BrokerConstants.java | 57 ++ .../tc/qmq/meta/BrokerAcquireMetaRequest.java | 50 ++ .../BrokerAcquireMetaRequestSerializer.java | 34 ++ .../qmq/meta/BrokerAcquireMetaResponse.java | 60 +++ .../BrokerAcquireMetaResponseSerializer.java | 36 ++ .../tc/qmq/meta/BrokerRegisterRequest.java | 80 +++ .../meta/BrokerRegisterRequestSerializer.java | 40 ++ .../tc/qmq/meta/BrokerRegisterResponse.java | 35 ++ .../tc/qmq/meta/BrokerRegisterService.java | 194 +++++++ .../qunar/tc/qmq/meta/BrokerRequestType.java | 35 ++ .../java/qunar/tc/qmq/meta/BrokerRole.java | 58 ++ .../main/java/qunar/tc/qmq/monitor/QMon.java | 393 ++++++++++++++ .../tc/qmq/netty/ConnectionEventHandler.java | 29 + .../qunar/tc/qmq/netty/ConnectionHandler.java | 46 ++ .../netty/DefaultConnectionEventHandler.java | 53 ++ .../tc/qmq/netty/NettyRequestExecutor.java | 110 ++++ .../tc/qmq/netty/NettyRequestProcessor.java | 33 ++ .../java/qunar/tc/qmq/netty/NettyServer.java | 97 ++++ .../tc/qmq/netty/NettyServerHandler.java | 82 +++ .../tc/qmq/service/HeartbeatManager.java | 63 +++ qmq-server/pom.xml | 72 +++ .../java/qunar/tc/qmq/base/ConsumerGroup.java | 72 +++ .../qunar/tc/qmq/base/ConsumerSequence.java | 92 ++++ .../qunar/tc/qmq/base/PullMessageResult.java | 69 +++ .../java/qunar/tc/qmq/base/ReceiveResult.java | 61 +++ .../qunar/tc/qmq/base/ReceivingMessage.java | 69 +++ .../tc/qmq/base/WritePutActionResult.java | 39 ++ .../qmq/consumer/ConsumerSequenceManager.java | 286 ++++++++++ .../tc/qmq/consumer/OfflineActionHandler.java | 59 +++ .../qunar/tc/qmq/consumer/OfflineTask.java | 88 ++++ .../java/qunar/tc/qmq/consumer/RetryTask.java | 139 +++++ .../qunar/tc/qmq/consumer/Subscriber.java | 139 +++++ .../qmq/consumer/SubscriberStatusChecker.java | 233 ++++++++ .../qunar/tc/qmq/container/Bootstrap.java | 27 + .../qunar/tc/qmq/lag/ConsumerLagService.java | 73 +++ .../processor/AbstractRequestProcessor.java | 32 ++ .../tc/qmq/processor/AckMessageProcessor.java | 184 +++++++ .../tc/qmq/processor/AckMessageWorker.java | 58 ++ .../BrokerConnectionEventHandler.java | 50 ++ .../qmq/processor/GetQueueCountProcessor.java | 99 ++++ .../qmq/processor/PullMessageProcessor.java | 371 +++++++++++++ .../tc/qmq/processor/PullMessageWorker.java | 104 ++++ .../qmq/processor/SendMessageProcessor.java | 114 ++++ .../tc/qmq/processor/SendMessageWorker.java | 261 +++++++++ .../tc/qmq/processor/filters/Invoker.java | 27 + .../qmq/processor/filters/ReceiveFilter.java | 26 + .../processor/filters/ReceiveFilterChain.java | 61 +++ .../qmq/processor/filters/ValidateFilter.java | 35 ++ .../qunar/tc/qmq/startup/ServerWrapper.java | 275 ++++++++++ .../java/qunar/tc/qmq/stats/BrokerStats.java | 56 ++ .../tc/qmq/stats/PerMinuteDeltaCounter.java | 74 +++ .../tc/qmq/store/MessageStoreWrapper.java | 364 +++++++++++++ .../main/java/qunar/tc/qmq/store/Tags.java | 89 ++++ .../sync/master/AbstractLogSyncWorker.java | 184 +++++++ .../qmq/sync/master/ActionLogSyncWorker.java | 50 ++ .../qmq/sync/master/HeartbeatSyncWorker.java | 78 +++ .../sync/master/MasterSyncNettyServer.java | 56 ++ .../qmq/sync/master/MessageLogSyncWorker.java | 72 +++ .../sync/master/SyncCheckpointProcessor.java | 77 +++ .../tc/qmq/sync/master/SyncLogProcessor.java | 120 +++++ .../tc/qmq/sync/master/SyncProcessor.java | 27 + .../tc/qmq/sync/master/SyncRequestEntry.java | 49 ++ .../tc/qmq/utils/ConsumerGroupUtils.java | 30 ++ .../qunar/tc/qmq/utils/HeaderSerializer.java | 54 ++ .../qunar/tc/qmq/utils/ServerTimerUtil.java | 37 ++ qmq-store/pom.xml | 65 +++ qmq-store/src/main/c/MmapUtil.c | 15 + qmq-store/src/main/c/MmapUtil.h | 20 + .../main/java/qunar/tc/qmq/store/Action.java | 33 ++ .../qunar/tc/qmq/store/ActionCheckpoint.java | 57 ++ .../tc/qmq/store/ActionCheckpointSerde.java | 188 +++++++ .../java/qunar/tc/qmq/store/ActionLog.java | 231 ++++++++ .../tc/qmq/store/ActionLogIterateService.java | 137 +++++ .../qunar/tc/qmq/store/ActionLogVisitor.java | 128 +++++ .../tc/qmq/store/ActionReaderWriter.java | 29 + .../java/qunar/tc/qmq/store/ActionType.java | 69 +++ .../tc/qmq/store/AppendMessageResult.java | 69 +++ .../tc/qmq/store/AppendMessageStatus.java | 30 ++ .../qunar/tc/qmq/store/CheckpointLoader.java | 27 + .../qunar/tc/qmq/store/CheckpointManager.java | 460 ++++++++++++++++ .../qunar/tc/qmq/store/CheckpointStore.java | 139 +++++ .../java/qunar/tc/qmq/store/ConsumeQueue.java | 99 ++++ .../tc/qmq/store/ConsumeQueueManager.java | 74 +++ .../tc/qmq/store/ConsumerGroupProgress.java | 68 +++ .../java/qunar/tc/qmq/store/ConsumerLog.java | 236 +++++++++ .../qunar/tc/qmq/store/ConsumerLogEntry.java | 68 +++ .../tc/qmq/store/ConsumerLogFlusher.java | 109 ++++ .../tc/qmq/store/ConsumerLogManager.java | 189 +++++++ .../tc/qmq/store/ConsumerLogWroteEvent.java | 39 ++ .../qunar/tc/qmq/store/ConsumerProgress.java | 109 ++++ .../java/qunar/tc/qmq/store/DataTransfer.java | 104 ++++ .../qunar/tc/qmq/store/DefaultStorage.java | 481 +++++++++++++++++ .../java/qunar/tc/qmq/store/FlushHook.java | 25 + .../qunar/tc/qmq/store/GetMessageResult.java | 119 +++++ .../qunar/tc/qmq/store/GetMessageStatus.java | 33 ++ .../qunar/tc/qmq/store/GroupAndSubject.java | 51 ++ .../java/qunar/tc/qmq/store/LogManager.java | 349 ++++++++++++ .../java/qunar/tc/qmq/store/LogSegment.java | 229 ++++++++ .../tc/qmq/store/LogSegmentValidator.java | 48 ++ .../java/qunar/tc/qmq/store/MagicCode.java | 37 ++ .../qunar/tc/qmq/store/MagicCodeSupport.java | 31 ++ .../tc/qmq/store/MaxAckedPullLogSequence.java | 75 +++ .../qmq/store/MaxPulledMessageSequence.java | 67 +++ .../qunar/tc/qmq/store/MessageAppender.java | 27 + .../qunar/tc/qmq/store/MessageCheckpoint.java | 61 +++ .../tc/qmq/store/MessageCheckpointSerde.java | 87 +++ .../qunar/tc/qmq/store/MessageFilter.java | 25 + .../java/qunar/tc/qmq/store/MessageLog.java | 282 ++++++++++ .../qmq/store/MessageLogIterateService.java | 133 +++++ .../qunar/tc/qmq/store/MessageLogMeta.java | 76 +++ .../tc/qmq/store/MessageLogMetaVisitor.java | 144 +++++ .../qunar/tc/qmq/store/MessageSequence.java | 39 ++ .../java/qunar/tc/qmq/store/MmapUtil.java | 60 +++ .../java/qunar/tc/qmq/store/OffsetBound.java | 39 ++ .../java/qunar/tc/qmq/store/OffsetRange.java | 47 ++ .../tc/qmq/store/PeriodicFlushService.java | 83 +++ .../main/java/qunar/tc/qmq/store/PullLog.java | 191 +++++++ .../qunar/tc/qmq/store/PullLogFlusher.java | 110 ++++ .../qunar/tc/qmq/store/PullLogManager.java | 154 ++++++ .../qunar/tc/qmq/store/PullLogMessage.java | 47 ++ .../qunar/tc/qmq/store/PutMessageResult.java | 39 ++ .../qunar/tc/qmq/store/PutMessageStatus.java | 39 ++ .../qunar/tc/qmq/store/ReferenceObject.java | 72 +++ .../qunar/tc/qmq/store/SegmentBuffer.java | 72 +++ .../main/java/qunar/tc/qmq/store/Serde.java | 27 + .../java/qunar/tc/qmq/store/Snapshot.java | 46 ++ .../qunar/tc/qmq/store/SnapshotStore.java | 178 +++++++ .../main/java/qunar/tc/qmq/store/Storage.java | 92 ++++ .../qunar/tc/qmq/store/StorageConfig.java | 53 ++ .../qunar/tc/qmq/store/StorageConfigImpl.java | 127 +++++ .../java/qunar/tc/qmq/store/StoreUtils.java | 41 ++ .../tc/qmq/store/action/ActionEvent.java | 41 ++ .../store/action/ForeverOfflineAction.java | 74 +++ .../ForeverOfflineActionReaderWriter.java | 51 ++ .../qmq/store/action/MaxSequencesUpdater.java | 47 ++ .../qunar/tc/qmq/store/action/PullAction.java | 134 +++++ .../store/action/PullActionReaderWriter.java | 80 +++ .../tc/qmq/store/action/PullLogBuilder.java | 63 +++ .../tc/qmq/store/action/RangeAckAction.java | 88 ++++ .../action/RangeAckActionReaderWriter.java | 55 ++ .../store/event/FixedExecOrderEventBus.java | 75 +++ .../resources/META-INF/native/libmmaputil.so | Bin 0 -> 6146 bytes qmq-sync/pom.xml | 38 ++ .../tc/qmq/sync/AbstractSyncLogProcessor.java | 44 ++ .../qunar/tc/qmq/sync/DelaySyncRequest.java | 112 ++++ .../qunar/tc/qmq/sync/HeartbeatProcessor.java | 59 +++ .../tc/qmq/sync/MasterSlaveSyncManager.java | 118 +++++ .../qunar/tc/qmq/sync/SlaveSyncClient.java | 84 +++ .../qunar/tc/qmq/sync/SlaveSyncSender.java | 41 ++ .../tc/qmq/sync/SyncActionLogProcessor.java | 45 ++ .../qunar/tc/qmq/sync/SyncLogProcessor.java | 30 ++ .../tc/qmq/sync/SyncMessageLogProcessor.java | 45 ++ .../main/java/qunar/tc/qmq/sync/SyncType.java | 35 ++ qmq-tools/pom.xml | 33 ++ .../tc/qmq/tools/MetaManagementService.java | 51 ++ .../main/java/qunar/tc/qmq/tools/Tools.java | 50 ++ .../qmq/tools/command/AddBrokerCommand.java | 74 +++ .../tools/command/AddNewSubjectCommand.java | 58 ++ .../command/AddSubjectBrokerGroupCommand.java | 58 ++ .../command/ExtendSubjectRouteCommand.java | 62 +++ .../command/ListBrokerGroupsCommand.java | 47 ++ .../qmq/tools/command/ListBrokersCommand.java | 51 ++ .../command/ListSubjectRoutesCommand.java | 47 ++ .../RemoveSubjectBrokerGroupCommand.java | 58 ++ .../tools/command/ReplaceBrokerCommand.java | 75 +++ 589 files changed, 47697 insertions(+) create mode 100644 .gitignore create mode 100644 Makefile create mode 100644 README.md create mode 100644 docs/cn/arch.md create mode 100644 docs/cn/code.md create mode 100644 docs/cn/consumer.md create mode 100644 docs/cn/contributing.md create mode 100644 docs/cn/delay.md create mode 100644 docs/cn/design.md create mode 100644 docs/cn/ha.md create mode 100644 docs/cn/install.md create mode 100644 docs/cn/monitor.md create mode 100644 docs/cn/opensource.md create mode 100644 docs/cn/producer.md create mode 100644 docs/cn/quickstart.md create mode 100644 docs/cn/support.md create mode 100644 docs/cn/tag.md create mode 100644 docs/cn/trace.md create mode 100644 docs/images/arch1.png create mode 100644 docs/images/arch3.png create mode 100644 docs/images/arch4.png create mode 100644 docs/images/design1.png create mode 100644 docs/images/design2.png create mode 100644 docs/images/design3.png create mode 100644 docs/images/design4.png create mode 100644 docs/images/ha.png create mode 100644 docs/images/support1.png create mode 100644 pom.xml create mode 100644 qmq-api/pom.xml create mode 100644 qmq-api/src/main/java/qunar/tc/qmq/BaseConsumer.java create mode 100644 qmq-api/src/main/java/qunar/tc/qmq/Filter.java create mode 100644 qmq-api/src/main/java/qunar/tc/qmq/FilterAttachable.java create mode 100644 qmq-api/src/main/java/qunar/tc/qmq/IdempotentAttachable.java create mode 100644 qmq-api/src/main/java/qunar/tc/qmq/IdempotentChecker.java create mode 100644 qmq-api/src/main/java/qunar/tc/qmq/ListenerHolder.java create mode 100644 qmq-api/src/main/java/qunar/tc/qmq/Message.java create mode 100644 qmq-api/src/main/java/qunar/tc/qmq/MessageConsumer.java create mode 100644 qmq-api/src/main/java/qunar/tc/qmq/MessageListener.java create mode 100644 qmq-api/src/main/java/qunar/tc/qmq/MessageProducer.java create mode 100644 qmq-api/src/main/java/qunar/tc/qmq/MessageSendStateListener.java create mode 100644 qmq-api/src/main/java/qunar/tc/qmq/NeedRetryException.java create mode 100644 qmq-api/src/main/java/qunar/tc/qmq/ProduceMessage.java create mode 100644 qmq-api/src/main/java/qunar/tc/qmq/PullConsumer.java create mode 100644 qmq-api/src/main/java/qunar/tc/qmq/SubscribeParam.java create mode 100644 qmq-api/src/main/java/qunar/tc/qmq/TagType.java create mode 100644 qmq-client/pom.xml create mode 100644 qmq-client/src/main/java/qunar/tc/qmq/broker/BrokerClusterInfo.java create mode 100644 qmq-client/src/main/java/qunar/tc/qmq/broker/BrokerGroupInfo.java create mode 100644 qmq-client/src/main/java/qunar/tc/qmq/broker/BrokerLoadBalance.java create mode 100644 qmq-client/src/main/java/qunar/tc/qmq/broker/BrokerService.java create mode 100644 qmq-client/src/main/java/qunar/tc/qmq/broker/CircuitBreaker.java create mode 100644 qmq-client/src/main/java/qunar/tc/qmq/broker/impl/BrokerServiceImpl.java create mode 100644 qmq-client/src/main/java/qunar/tc/qmq/broker/impl/PollBrokerLoadBalance.java create mode 100644 qmq-client/src/main/java/qunar/tc/qmq/common/AtomicConfig.java create mode 100644 qmq-client/src/main/java/qunar/tc/qmq/common/AtomicIntegerConfig.java create mode 100644 qmq-client/src/main/java/qunar/tc/qmq/common/ClientIdProvider.java create mode 100644 qmq-client/src/main/java/qunar/tc/qmq/common/ClientIdProviderFactory.java create mode 100644 qmq-client/src/main/java/qunar/tc/qmq/common/DefaultClientIdProvider.java create mode 100644 qmq-client/src/main/java/qunar/tc/qmq/common/MapKeyBuilder.java create mode 100644 qmq-client/src/main/java/qunar/tc/qmq/common/StatusSource.java create mode 100644 qmq-client/src/main/java/qunar/tc/qmq/common/SwitchWaiter.java create mode 100644 qmq-client/src/main/java/qunar/tc/qmq/common/TimerUtil.java create mode 100644 qmq-client/src/main/java/qunar/tc/qmq/config/NettyClientConfigManager.java create mode 100644 qmq-client/src/main/java/qunar/tc/qmq/config/PullSubjectsConfig.java create mode 100644 qmq-client/src/main/java/qunar/tc/qmq/consumer/BaseMessageHandler.java create mode 100644 qmq-client/src/main/java/qunar/tc/qmq/consumer/ConsumeMessage.java create mode 100644 qmq-client/src/main/java/qunar/tc/qmq/consumer/MessageConsumerProvider.java create mode 100644 qmq-client/src/main/java/qunar/tc/qmq/consumer/annotation/ConsumerAnnotationScanner.java create mode 100644 qmq-client/src/main/java/qunar/tc/qmq/consumer/annotation/EnableQmq.java create mode 100644 qmq-client/src/main/java/qunar/tc/qmq/consumer/annotation/GeneratedListener.java create mode 100644 qmq-client/src/main/java/qunar/tc/qmq/consumer/annotation/QmqClientBeanDefinitionParser.java create mode 100644 qmq-client/src/main/java/qunar/tc/qmq/consumer/annotation/QmqClientNamespaceHandler.java create mode 100644 qmq-client/src/main/java/qunar/tc/qmq/consumer/annotation/QmqConsumer.java create mode 100644 qmq-client/src/main/java/qunar/tc/qmq/consumer/annotation/QmqConsumerRegister.java create mode 100644 qmq-client/src/main/java/qunar/tc/qmq/consumer/exception/CreatePullConsumerException.java create mode 100644 qmq-client/src/main/java/qunar/tc/qmq/consumer/exception/DuplicateListenerException.java create mode 100644 qmq-client/src/main/java/qunar/tc/qmq/consumer/handler/IdempotentCheckerFilter.java create mode 100644 qmq-client/src/main/java/qunar/tc/qmq/consumer/handler/MessageDistributor.java create mode 100644 qmq-client/src/main/java/qunar/tc/qmq/consumer/handler/QTraceFilter.java create mode 100644 qmq-client/src/main/java/qunar/tc/qmq/consumer/idempotent/AbstractIdempotentChecker.java create mode 100644 qmq-client/src/main/java/qunar/tc/qmq/consumer/idempotent/JdbcIdempotentChecker.java create mode 100644 qmq-client/src/main/java/qunar/tc/qmq/consumer/idempotent/TransactionalJdbcIdempotentChecker.java create mode 100644 qmq-client/src/main/java/qunar/tc/qmq/consumer/pull/AbstractPullConsumer.java create mode 100644 qmq-client/src/main/java/qunar/tc/qmq/consumer/pull/AbstractPullEntry.java create mode 100644 qmq-client/src/main/java/qunar/tc/qmq/consumer/pull/AckEntry.java create mode 100644 qmq-client/src/main/java/qunar/tc/qmq/consumer/pull/AckHelper.java create mode 100644 qmq-client/src/main/java/qunar/tc/qmq/consumer/pull/AckHook.java create mode 100644 qmq-client/src/main/java/qunar/tc/qmq/consumer/pull/AckSendEntry.java create mode 100644 qmq-client/src/main/java/qunar/tc/qmq/consumer/pull/AckSendInfo.java create mode 100644 qmq-client/src/main/java/qunar/tc/qmq/consumer/pull/AckSendQueue.java create mode 100644 qmq-client/src/main/java/qunar/tc/qmq/consumer/pull/AckService.java create mode 100644 qmq-client/src/main/java/qunar/tc/qmq/consumer/pull/AlwaysPullStrategy.java create mode 100644 qmq-client/src/main/java/qunar/tc/qmq/consumer/pull/ConsumeParam.java create mode 100644 qmq-client/src/main/java/qunar/tc/qmq/consumer/pull/DefaultPullConsumer.java create mode 100644 qmq-client/src/main/java/qunar/tc/qmq/consumer/pull/DelayMessageService.java create mode 100644 qmq-client/src/main/java/qunar/tc/qmq/consumer/pull/PlainPullEntry.java create mode 100644 qmq-client/src/main/java/qunar/tc/qmq/consumer/pull/PullConsumerFactory.java create mode 100644 qmq-client/src/main/java/qunar/tc/qmq/consumer/pull/PullEntry.java create mode 100644 qmq-client/src/main/java/qunar/tc/qmq/consumer/pull/PullMessageFuture.java create mode 100644 qmq-client/src/main/java/qunar/tc/qmq/consumer/pull/PullParam.java create mode 100644 qmq-client/src/main/java/qunar/tc/qmq/consumer/pull/PullRegister.java create mode 100644 qmq-client/src/main/java/qunar/tc/qmq/consumer/pull/PullResult.java create mode 100644 qmq-client/src/main/java/qunar/tc/qmq/consumer/pull/PullService.java create mode 100644 qmq-client/src/main/java/qunar/tc/qmq/consumer/pull/PullStrategy.java create mode 100644 qmq-client/src/main/java/qunar/tc/qmq/consumer/pull/PulledMessage.java create mode 100644 qmq-client/src/main/java/qunar/tc/qmq/consumer/pull/PulledMessageFilter.java create mode 100644 qmq-client/src/main/java/qunar/tc/qmq/consumer/pull/PushConsumer.java create mode 100644 qmq-client/src/main/java/qunar/tc/qmq/consumer/pull/PushConsumerImpl.java create mode 100644 qmq-client/src/main/java/qunar/tc/qmq/consumer/pull/SendMessageBack.java create mode 100644 qmq-client/src/main/java/qunar/tc/qmq/consumer/pull/SendMessageBackImpl.java create mode 100644 qmq-client/src/main/java/qunar/tc/qmq/consumer/pull/WeightLoadBalance.java create mode 100644 qmq-client/src/main/java/qunar/tc/qmq/consumer/pull/WeightPullStrategy.java create mode 100644 qmq-client/src/main/java/qunar/tc/qmq/consumer/pull/exception/AckException.java create mode 100644 qmq-client/src/main/java/qunar/tc/qmq/consumer/pull/exception/PullException.java create mode 100644 qmq-client/src/main/java/qunar/tc/qmq/consumer/pull/exception/SendMessageBackException.java create mode 100644 qmq-client/src/main/java/qunar/tc/qmq/consumer/register/ConsumerRegister.java create mode 100644 qmq-client/src/main/java/qunar/tc/qmq/consumer/register/RegistParam.java create mode 100644 qmq-client/src/main/java/qunar/tc/qmq/metainfoclient/ConsumerStateChangedListener.java create mode 100644 qmq-client/src/main/java/qunar/tc/qmq/metainfoclient/MetaInfo.java create mode 100644 qmq-client/src/main/java/qunar/tc/qmq/metainfoclient/MetaInfoClient.java create mode 100644 qmq-client/src/main/java/qunar/tc/qmq/metainfoclient/MetaInfoClientHandler.java create mode 100644 qmq-client/src/main/java/qunar/tc/qmq/metainfoclient/MetaInfoClientNettyImpl.java create mode 100644 qmq-client/src/main/java/qunar/tc/qmq/metainfoclient/MetaInfoService.java create mode 100644 qmq-client/src/main/java/qunar/tc/qmq/producer/ConfigCenter.java create mode 100644 qmq-client/src/main/java/qunar/tc/qmq/producer/MessageProducerProvider.java create mode 100644 qmq-client/src/main/java/qunar/tc/qmq/producer/ProduceMessageImpl.java create mode 100644 qmq-client/src/main/java/qunar/tc/qmq/producer/QueueSender.java create mode 100644 qmq-client/src/main/java/qunar/tc/qmq/producer/RegistryResolver.java create mode 100644 qmq-client/src/main/java/qunar/tc/qmq/producer/SendErrorHandler.java create mode 100644 qmq-client/src/main/java/qunar/tc/qmq/producer/idgenerator/IdGenerator.java create mode 100644 qmq-client/src/main/java/qunar/tc/qmq/producer/idgenerator/TimestampAndHostIdGenerator.java create mode 100644 qmq-client/src/main/java/qunar/tc/qmq/producer/sender/AbstractRouterManager.java create mode 100644 qmq-client/src/main/java/qunar/tc/qmq/producer/sender/Connection.java create mode 100644 qmq-client/src/main/java/qunar/tc/qmq/producer/sender/MessageSenderGroup.java create mode 100644 qmq-client/src/main/java/qunar/tc/qmq/producer/sender/NettyConnection.java create mode 100644 qmq-client/src/main/java/qunar/tc/qmq/producer/sender/NettyProducerClient.java create mode 100644 qmq-client/src/main/java/qunar/tc/qmq/producer/sender/NettyRouter.java create mode 100644 qmq-client/src/main/java/qunar/tc/qmq/producer/sender/NettyRouterManager.java create mode 100644 qmq-client/src/main/java/qunar/tc/qmq/producer/sender/NopRoute.java create mode 100644 qmq-client/src/main/java/qunar/tc/qmq/producer/sender/RPCQueueSender.java create mode 100644 qmq-client/src/main/java/qunar/tc/qmq/producer/sender/Route.java create mode 100644 qmq-client/src/main/java/qunar/tc/qmq/producer/sender/Router.java create mode 100644 qmq-client/src/main/java/qunar/tc/qmq/producer/sender/RouterManager.java create mode 100644 qmq-client/src/main/java/qunar/tc/qmq/tracing/QmqMessageExtractAdapter.java create mode 100644 qmq-client/src/main/java/qunar/tc/qmq/tracing/QmqMessageInjectAdapter.java create mode 100644 qmq-client/src/main/java/qunar/tc/qmq/tracing/TraceUtil.java create mode 100644 qmq-client/src/main/resources/META-INF/qmq-2.0.0.xsd create mode 100644 qmq-client/src/main/resources/META-INF/spring.handlers create mode 100644 qmq-client/src/main/resources/META-INF/spring.schemas create mode 100644 qmq-common/pom.xml create mode 100644 qmq-common/src/main/java/qunar/tc/qmq/base/BaseMessage.java create mode 100644 qmq-common/src/main/java/qunar/tc/qmq/base/ClientRequestType.java create mode 100644 qmq-common/src/main/java/qunar/tc/qmq/base/LargeStringUtil.java create mode 100644 qmq-common/src/main/java/qunar/tc/qmq/base/MessageHeader.java create mode 100644 qmq-common/src/main/java/qunar/tc/qmq/base/OnOfflineState.java create mode 100644 qmq-common/src/main/java/qunar/tc/qmq/base/RawMessage.java create mode 100644 qmq-common/src/main/java/qunar/tc/qmq/batch/BatchExecutor.java create mode 100644 qmq-common/src/main/java/qunar/tc/qmq/batch/Processor.java create mode 100644 qmq-common/src/main/java/qunar/tc/qmq/common/ClientType.java create mode 100644 qmq-common/src/main/java/qunar/tc/qmq/common/Disposable.java create mode 100644 qmq-common/src/main/java/qunar/tc/qmq/concurrent/NamedThreadFactory.java create mode 100644 qmq-common/src/main/java/qunar/tc/qmq/configuration/DynamicConfig.java create mode 100644 qmq-common/src/main/java/qunar/tc/qmq/configuration/DynamicConfigFactory.java create mode 100644 qmq-common/src/main/java/qunar/tc/qmq/configuration/DynamicConfigLoader.java create mode 100644 qmq-common/src/main/java/qunar/tc/qmq/configuration/Listener.java create mode 100644 qmq-common/src/main/java/qunar/tc/qmq/configuration/local/ConfigWatcher.java create mode 100644 qmq-common/src/main/java/qunar/tc/qmq/configuration/local/LocalDynamicConfig.java create mode 100644 qmq-common/src/main/java/qunar/tc/qmq/configuration/local/LocalDynamicConfigFactory.java create mode 100644 qmq-common/src/main/java/qunar/tc/qmq/meta/BrokerCluster.java create mode 100644 qmq-common/src/main/java/qunar/tc/qmq/meta/BrokerGroup.java create mode 100644 qmq-common/src/main/java/qunar/tc/qmq/meta/BrokerGroupKind.java create mode 100644 qmq-common/src/main/java/qunar/tc/qmq/meta/BrokerState.java create mode 100644 qmq-common/src/main/java/qunar/tc/qmq/meta/MetaServerLocator.java create mode 100644 qmq-common/src/main/java/qunar/tc/qmq/metrics/Metrics.java create mode 100644 qmq-common/src/main/java/qunar/tc/qmq/metrics/MetricsConstants.java create mode 100644 qmq-common/src/main/java/qunar/tc/qmq/metrics/MockRegistry.java create mode 100644 qmq-common/src/main/java/qunar/tc/qmq/metrics/QmqCounter.java create mode 100644 qmq-common/src/main/java/qunar/tc/qmq/metrics/QmqMeter.java create mode 100644 qmq-common/src/main/java/qunar/tc/qmq/metrics/QmqMetricRegistry.java create mode 100644 qmq-common/src/main/java/qunar/tc/qmq/metrics/QmqTimer.java create mode 100644 qmq-common/src/main/java/qunar/tc/qmq/service/exceptions/BlockMessageException.java create mode 100644 qmq-common/src/main/java/qunar/tc/qmq/service/exceptions/DuplicateMessageException.java create mode 100644 qmq-common/src/main/java/qunar/tc/qmq/service/exceptions/MessageException.java create mode 100644 qmq-common/src/main/java/qunar/tc/qmq/utils/CharsetUtils.java create mode 100644 qmq-common/src/main/java/qunar/tc/qmq/utils/Checksums.java create mode 100644 qmq-common/src/main/java/qunar/tc/qmq/utils/Crc32.java create mode 100644 qmq-common/src/main/java/qunar/tc/qmq/utils/DelayUtil.java create mode 100644 qmq-common/src/main/java/qunar/tc/qmq/utils/Flags.java create mode 100644 qmq-common/src/main/java/qunar/tc/qmq/utils/ListUtils.java create mode 100644 qmq-common/src/main/java/qunar/tc/qmq/utils/NetworkUtils.java create mode 100644 qmq-common/src/main/java/qunar/tc/qmq/utils/ObjectUtils.java create mode 100644 qmq-common/src/main/java/qunar/tc/qmq/utils/PayloadHolderUtils.java create mode 100644 qmq-common/src/main/java/qunar/tc/qmq/utils/PidUtil.java create mode 100644 qmq-common/src/main/java/qunar/tc/qmq/utils/RetrySubjectUtils.java create mode 100644 qmq-common/src/main/resources/META-INF/services/qunar.tc.qmq.configuration.DynamicConfigFactory create mode 100644 qmq-delay-server/pom.xml create mode 100644 qmq-delay-server/src/main/java/qunar/tc/qmq/delay/DefaultDelayLogFacade.java create mode 100644 qmq-delay-server/src/main/java/qunar/tc/qmq/delay/DelayLogFacade.java create mode 100644 qmq-delay-server/src/main/java/qunar/tc/qmq/delay/EventListener.java create mode 100644 qmq-delay-server/src/main/java/qunar/tc/qmq/delay/LogFlusher.java create mode 100644 qmq-delay-server/src/main/java/qunar/tc/qmq/delay/MessageIterateEventListener.java create mode 100644 qmq-delay-server/src/main/java/qunar/tc/qmq/delay/MessageLogReplayer.java create mode 100644 qmq-delay-server/src/main/java/qunar/tc/qmq/delay/ScheduleIndex.java create mode 100644 qmq-delay-server/src/main/java/qunar/tc/qmq/delay/Switchable.java create mode 100644 qmq-delay-server/src/main/java/qunar/tc/qmq/delay/base/AppendException.java create mode 100644 qmq-delay-server/src/main/java/qunar/tc/qmq/delay/base/GroupSendException.java create mode 100644 qmq-delay-server/src/main/java/qunar/tc/qmq/delay/base/LongHashSet.java create mode 100644 qmq-delay-server/src/main/java/qunar/tc/qmq/delay/base/ReceivedDelayMessage.java create mode 100644 qmq-delay-server/src/main/java/qunar/tc/qmq/delay/base/ReceivedResult.java create mode 100644 qmq-delay-server/src/main/java/qunar/tc/qmq/delay/base/SegmentBufferExtend.java create mode 100644 qmq-delay-server/src/main/java/qunar/tc/qmq/delay/cleaner/LogCleaner.java create mode 100644 qmq-delay-server/src/main/java/qunar/tc/qmq/delay/config/DefaultStoreConfiguration.java create mode 100644 qmq-delay-server/src/main/java/qunar/tc/qmq/delay/config/StoreConfiguration.java create mode 100644 qmq-delay-server/src/main/java/qunar/tc/qmq/delay/container/Bootstrap.java create mode 100644 qmq-delay-server/src/main/java/qunar/tc/qmq/delay/meta/BrokerRoleManager.java create mode 100644 qmq-delay-server/src/main/java/qunar/tc/qmq/delay/monitor/QMon.java create mode 100644 qmq-delay-server/src/main/java/qunar/tc/qmq/delay/receiver/Invoker.java create mode 100644 qmq-delay-server/src/main/java/qunar/tc/qmq/delay/receiver/ReceivedDelayMessageProcessor.java create mode 100644 qmq-delay-server/src/main/java/qunar/tc/qmq/delay/receiver/Receiver.java create mode 100644 qmq-delay-server/src/main/java/qunar/tc/qmq/delay/receiver/filter/Filter.java create mode 100644 qmq-delay-server/src/main/java/qunar/tc/qmq/delay/receiver/filter/OverDelayException.java create mode 100644 qmq-delay-server/src/main/java/qunar/tc/qmq/delay/receiver/filter/OverDelayFilter.java create mode 100644 qmq-delay-server/src/main/java/qunar/tc/qmq/delay/receiver/filter/PastDelayFilter.java create mode 100644 qmq-delay-server/src/main/java/qunar/tc/qmq/delay/receiver/filter/ReceiveFilterChain.java create mode 100644 qmq-delay-server/src/main/java/qunar/tc/qmq/delay/receiver/filter/ValidateFilter.java create mode 100644 qmq-delay-server/src/main/java/qunar/tc/qmq/delay/sender/DelayProcessor.java create mode 100644 qmq-delay-server/src/main/java/qunar/tc/qmq/delay/sender/NettySender.java create mode 100644 qmq-delay-server/src/main/java/qunar/tc/qmq/delay/sender/Sender.java create mode 100644 qmq-delay-server/src/main/java/qunar/tc/qmq/delay/sender/SenderExecutor.java create mode 100644 qmq-delay-server/src/main/java/qunar/tc/qmq/delay/sender/SenderGroup.java create mode 100644 qmq-delay-server/src/main/java/qunar/tc/qmq/delay/sender/SenderProcessor.java create mode 100644 qmq-delay-server/src/main/java/qunar/tc/qmq/delay/startup/ServerWrapper.java create mode 100644 qmq-delay-server/src/main/java/qunar/tc/qmq/delay/store/DefaultDelaySegmentValidator.java create mode 100644 qmq-delay-server/src/main/java/qunar/tc/qmq/delay/store/DelaySegmentValidator.java create mode 100644 qmq-delay-server/src/main/java/qunar/tc/qmq/delay/store/IterateOffsetManager.java create mode 100644 qmq-delay-server/src/main/java/qunar/tc/qmq/delay/store/ScheduleLogValidatorSupport.java create mode 100644 qmq-delay-server/src/main/java/qunar/tc/qmq/delay/store/VisitorAccessor.java create mode 100644 qmq-delay-server/src/main/java/qunar/tc/qmq/delay/store/appender/DispatchLogAppender.java create mode 100644 qmq-delay-server/src/main/java/qunar/tc/qmq/delay/store/appender/LogAppender.java create mode 100644 qmq-delay-server/src/main/java/qunar/tc/qmq/delay/store/appender/ScheduleSetAppender.java create mode 100644 qmq-delay-server/src/main/java/qunar/tc/qmq/delay/store/log/AbstractDelayLog.java create mode 100644 qmq-delay-server/src/main/java/qunar/tc/qmq/delay/store/log/AbstractDelaySegment.java create mode 100644 qmq-delay-server/src/main/java/qunar/tc/qmq/delay/store/log/AbstractDelaySegmentContainer.java create mode 100644 qmq-delay-server/src/main/java/qunar/tc/qmq/delay/store/log/DelaySegment.java create mode 100644 qmq-delay-server/src/main/java/qunar/tc/qmq/delay/store/log/DispatchLog.java create mode 100644 qmq-delay-server/src/main/java/qunar/tc/qmq/delay/store/log/DispatchLogSegment.java create mode 100644 qmq-delay-server/src/main/java/qunar/tc/qmq/delay/store/log/DispatchLogSegmentContainer.java create mode 100644 qmq-delay-server/src/main/java/qunar/tc/qmq/delay/store/log/Log.java create mode 100644 qmq-delay-server/src/main/java/qunar/tc/qmq/delay/store/log/MessageLog.java create mode 100644 qmq-delay-server/src/main/java/qunar/tc/qmq/delay/store/log/MessageSegmentContainer.java create mode 100644 qmq-delay-server/src/main/java/qunar/tc/qmq/delay/store/log/ScheduleLog.java create mode 100644 qmq-delay-server/src/main/java/qunar/tc/qmq/delay/store/log/ScheduleOffsetResolver.java create mode 100644 qmq-delay-server/src/main/java/qunar/tc/qmq/delay/store/log/ScheduleSet.java create mode 100644 qmq-delay-server/src/main/java/qunar/tc/qmq/delay/store/log/ScheduleSetSegment.java create mode 100644 qmq-delay-server/src/main/java/qunar/tc/qmq/delay/store/log/ScheduleSetSegmentContainer.java create mode 100644 qmq-delay-server/src/main/java/qunar/tc/qmq/delay/store/log/SegmentContainer.java create mode 100644 qmq-delay-server/src/main/java/qunar/tc/qmq/delay/store/model/AppendDispatchRecordResult.java create mode 100644 qmq-delay-server/src/main/java/qunar/tc/qmq/delay/store/model/AppendLogResult.java create mode 100644 qmq-delay-server/src/main/java/qunar/tc/qmq/delay/store/model/AppendMessageRecordResult.java create mode 100644 qmq-delay-server/src/main/java/qunar/tc/qmq/delay/store/model/AppendRecordResult.java create mode 100644 qmq-delay-server/src/main/java/qunar/tc/qmq/delay/store/model/AppendScheduleLogRecordResult.java create mode 100644 qmq-delay-server/src/main/java/qunar/tc/qmq/delay/store/model/DispatchLogRecord.java create mode 100644 qmq-delay-server/src/main/java/qunar/tc/qmq/delay/store/model/LogRecord.java create mode 100644 qmq-delay-server/src/main/java/qunar/tc/qmq/delay/store/model/LogRecordHeader.java create mode 100644 qmq-delay-server/src/main/java/qunar/tc/qmq/delay/store/model/MessageLogAttrEnum.java create mode 100644 qmq-delay-server/src/main/java/qunar/tc/qmq/delay/store/model/MessageLogRecord.java create mode 100644 qmq-delay-server/src/main/java/qunar/tc/qmq/delay/store/model/NopeRecordResult.java create mode 100644 qmq-delay-server/src/main/java/qunar/tc/qmq/delay/store/model/RawMessageExtend.java create mode 100644 qmq-delay-server/src/main/java/qunar/tc/qmq/delay/store/model/RecordResult.java create mode 100644 qmq-delay-server/src/main/java/qunar/tc/qmq/delay/store/model/ScheduleSetRecord.java create mode 100644 qmq-delay-server/src/main/java/qunar/tc/qmq/delay/store/model/ScheduleSetSequence.java create mode 100644 qmq-delay-server/src/main/java/qunar/tc/qmq/delay/store/visitor/AbstractLogVisitor.java create mode 100644 qmq-delay-server/src/main/java/qunar/tc/qmq/delay/store/visitor/DelayMessageLogVisitor.java create mode 100644 qmq-delay-server/src/main/java/qunar/tc/qmq/delay/store/visitor/DispatchLogVisitor.java create mode 100644 qmq-delay-server/src/main/java/qunar/tc/qmq/delay/store/visitor/LogVisitor.java create mode 100644 qmq-delay-server/src/main/java/qunar/tc/qmq/delay/store/visitor/ScheduleIndexVisitor.java create mode 100644 qmq-delay-server/src/main/java/qunar/tc/qmq/delay/sync/master/AbstractLogSyncWorker.java create mode 100644 qmq-delay-server/src/main/java/qunar/tc/qmq/delay/sync/master/DelaySyncRequestProcessor.java create mode 100644 qmq-delay-server/src/main/java/qunar/tc/qmq/delay/sync/master/DispatchLogSyncWorker.java create mode 100644 qmq-delay-server/src/main/java/qunar/tc/qmq/delay/sync/master/HeartbeatSyncWorker.java create mode 100644 qmq-delay-server/src/main/java/qunar/tc/qmq/delay/sync/master/MasterSyncNettyServer.java create mode 100644 qmq-delay-server/src/main/java/qunar/tc/qmq/delay/sync/master/MessageLogSyncWorker.java create mode 100644 qmq-delay-server/src/main/java/qunar/tc/qmq/delay/sync/slave/DispatchLogSyncProcessor.java create mode 100644 qmq-delay-server/src/main/java/qunar/tc/qmq/delay/sync/slave/HeartBeatProcessor.java create mode 100644 qmq-delay-server/src/main/java/qunar/tc/qmq/delay/sync/slave/MessageLogSyncProcessor.java create mode 100644 qmq-delay-server/src/main/java/qunar/tc/qmq/delay/sync/slave/SlaveSynchronizer.java create mode 100644 qmq-delay-server/src/main/java/qunar/tc/qmq/delay/sync/slave/SyncLogProcessor.java create mode 100644 qmq-delay-server/src/main/java/qunar/tc/qmq/delay/wheel/HashedWheelTimer.java create mode 100644 qmq-delay-server/src/main/java/qunar/tc/qmq/delay/wheel/WheelLoadCursor.java create mode 100644 qmq-delay-server/src/main/java/qunar/tc/qmq/delay/wheel/WheelTickManager.java create mode 100644 qmq-dist/assembly/bin.xml create mode 100755 qmq-dist/bin/base.sh create mode 100644 qmq-dist/bin/broker-env.sh create mode 100644 qmq-dist/bin/broker.sh create mode 100755 qmq-dist/bin/delay-env.sh create mode 100755 qmq-dist/bin/delay.sh create mode 100755 qmq-dist/bin/metaserver-env.sh create mode 100755 qmq-dist/bin/metaserver.sh create mode 100755 qmq-dist/bin/tools-env.sh create mode 100755 qmq-dist/bin/tools.sh create mode 100644 qmq-dist/conf/broker.properties create mode 100644 qmq-dist/conf/datasource.properties create mode 100644 qmq-dist/conf/delay.properties create mode 100644 qmq-dist/conf/logback.xml create mode 100644 qmq-dist/conf/metaserver.properties create mode 100644 qmq-dist/conf/valid-api-tokens.properties create mode 100644 qmq-dist/pom.xml create mode 100644 qmq-metaserver/pom.xml create mode 100644 qmq-metaserver/sql/init.sql create mode 100644 qmq-metaserver/src/main/java/qunar/tc/qmq/meta/cache/AliveClientManager.java create mode 100644 qmq-metaserver/src/main/java/qunar/tc/qmq/meta/cache/CachedMetaInfoManager.java create mode 100644 qmq-metaserver/src/main/java/qunar/tc/qmq/meta/cache/CachedOfflineStateManager.java create mode 100644 qmq-metaserver/src/main/java/qunar/tc/qmq/meta/container/WebContainerListener.java create mode 100644 qmq-metaserver/src/main/java/qunar/tc/qmq/meta/loadbalance/AbstractLoadBalance.java create mode 100644 qmq-metaserver/src/main/java/qunar/tc/qmq/meta/loadbalance/LoadBalance.java create mode 100644 qmq-metaserver/src/main/java/qunar/tc/qmq/meta/loadbalance/RandomLoadBalance.java create mode 100644 qmq-metaserver/src/main/java/qunar/tc/qmq/meta/management/ActionResult.java create mode 100644 qmq-metaserver/src/main/java/qunar/tc/qmq/meta/management/AddBrokerAction.java create mode 100644 qmq-metaserver/src/main/java/qunar/tc/qmq/meta/management/AddNewSubjectAction.java create mode 100644 qmq-metaserver/src/main/java/qunar/tc/qmq/meta/management/AddSubjectBrokerGroupAction.java create mode 100644 qmq-metaserver/src/main/java/qunar/tc/qmq/meta/management/ExtendSubjectRouteAction.java create mode 100644 qmq-metaserver/src/main/java/qunar/tc/qmq/meta/management/ListBrokerGroupsAction.java create mode 100644 qmq-metaserver/src/main/java/qunar/tc/qmq/meta/management/ListBrokersAction.java create mode 100644 qmq-metaserver/src/main/java/qunar/tc/qmq/meta/management/ListSubjectRoutesAction.java create mode 100644 qmq-metaserver/src/main/java/qunar/tc/qmq/meta/management/MetaManagementAction.java create mode 100644 qmq-metaserver/src/main/java/qunar/tc/qmq/meta/management/MetaManagementActionSupplier.java create mode 100644 qmq-metaserver/src/main/java/qunar/tc/qmq/meta/management/QuerySubjectRouteAction.java create mode 100644 qmq-metaserver/src/main/java/qunar/tc/qmq/meta/management/RemoveSubjectBrokerGroupAction.java create mode 100644 qmq-metaserver/src/main/java/qunar/tc/qmq/meta/management/ReplaceBrokerAction.java create mode 100644 qmq-metaserver/src/main/java/qunar/tc/qmq/meta/management/TokenVerificationAction.java create mode 100644 qmq-metaserver/src/main/java/qunar/tc/qmq/meta/model/BrokerMeta.java create mode 100644 qmq-metaserver/src/main/java/qunar/tc/qmq/meta/model/ClientMetaInfo.java create mode 100644 qmq-metaserver/src/main/java/qunar/tc/qmq/meta/model/ClientOfflineState.java create mode 100644 qmq-metaserver/src/main/java/qunar/tc/qmq/meta/model/GroupedConsumer.java create mode 100644 qmq-metaserver/src/main/java/qunar/tc/qmq/meta/model/ReadonlyBrokerGroupSetting.java create mode 100644 qmq-metaserver/src/main/java/qunar/tc/qmq/meta/model/SubjectInfo.java create mode 100644 qmq-metaserver/src/main/java/qunar/tc/qmq/meta/model/SubjectRoute.java create mode 100644 qmq-metaserver/src/main/java/qunar/tc/qmq/meta/monitor/QMon.java create mode 100644 qmq-metaserver/src/main/java/qunar/tc/qmq/meta/processor/BrokerAcquireMetaProcessor.java create mode 100644 qmq-metaserver/src/main/java/qunar/tc/qmq/meta/processor/BrokerRegisterProcessor.java create mode 100644 qmq-metaserver/src/main/java/qunar/tc/qmq/meta/processor/ClientRegisterProcessor.java create mode 100644 qmq-metaserver/src/main/java/qunar/tc/qmq/meta/processor/ClientRegisterWorker.java create mode 100644 qmq-metaserver/src/main/java/qunar/tc/qmq/meta/route/SubjectConsumerService.java create mode 100644 qmq-metaserver/src/main/java/qunar/tc/qmq/meta/route/SubjectRegisterService.java create mode 100644 qmq-metaserver/src/main/java/qunar/tc/qmq/meta/route/SubjectRouter.java create mode 100644 qmq-metaserver/src/main/java/qunar/tc/qmq/meta/route/impl/DefaultSubjectRouter.java create mode 100644 qmq-metaserver/src/main/java/qunar/tc/qmq/meta/route/impl/DelayRouter.java create mode 100644 qmq-metaserver/src/main/java/qunar/tc/qmq/meta/route/impl/SubjectConsumerServiceImpl.java create mode 100644 qmq-metaserver/src/main/java/qunar/tc/qmq/meta/startup/Bootstrap.java create mode 100644 qmq-metaserver/src/main/java/qunar/tc/qmq/meta/startup/ServerWrapper.java create mode 100644 qmq-metaserver/src/main/java/qunar/tc/qmq/meta/store/BrokerStore.java create mode 100644 qmq-metaserver/src/main/java/qunar/tc/qmq/meta/store/ClientMetaInfoStore.java create mode 100644 qmq-metaserver/src/main/java/qunar/tc/qmq/meta/store/ClientOfflineStore.java create mode 100644 qmq-metaserver/src/main/java/qunar/tc/qmq/meta/store/JdbcTemplateHolder.java create mode 100644 qmq-metaserver/src/main/java/qunar/tc/qmq/meta/store/Store.java create mode 100644 qmq-metaserver/src/main/java/qunar/tc/qmq/meta/store/impl/BrokerStoreImpl.java create mode 100644 qmq-metaserver/src/main/java/qunar/tc/qmq/meta/store/impl/ClientMetaInfoStoreImpl.java create mode 100644 qmq-metaserver/src/main/java/qunar/tc/qmq/meta/store/impl/ClientOfflineStoreImpl.java create mode 100644 qmq-metaserver/src/main/java/qunar/tc/qmq/meta/store/impl/DatabaseStore.java create mode 100644 qmq-metaserver/src/main/java/qunar/tc/qmq/meta/store/impl/Serializer.java create mode 100644 qmq-metaserver/src/main/java/qunar/tc/qmq/meta/utils/ClientLogUtils.java create mode 100644 qmq-metaserver/src/main/java/qunar/tc/qmq/meta/utils/SubjectUtils.java create mode 100644 qmq-metaserver/src/main/java/qunar/tc/qmq/meta/web/JsonResult.java create mode 100644 qmq-metaserver/src/main/java/qunar/tc/qmq/meta/web/MetaManagementServlet.java create mode 100644 qmq-metaserver/src/main/java/qunar/tc/qmq/meta/web/MetaServerAddressSupplierServlet.java create mode 100644 qmq-metaserver/src/main/java/qunar/tc/qmq/meta/web/OnOfflineServlet.java create mode 100644 qmq-metaserver/src/main/java/qunar/tc/qmq/meta/web/ResultStatus.java create mode 100644 qmq-metaserver/src/main/java/qunar/tc/qmq/meta/web/SubjectConsumerServlet.java create mode 100644 qmq-metrics-prometheus/pom.xml create mode 100644 qmq-metrics-prometheus/src/main/java/qunar/tc/qmq/metrics/prometheus/PrometheusQmqCounter.java create mode 100644 qmq-metrics-prometheus/src/main/java/qunar/tc/qmq/metrics/prometheus/PrometheusQmqGauge.java create mode 100644 qmq-metrics-prometheus/src/main/java/qunar/tc/qmq/metrics/prometheus/PrometheusQmqMeter.java create mode 100644 qmq-metrics-prometheus/src/main/java/qunar/tc/qmq/metrics/prometheus/PrometheusQmqMetricRegistry.java create mode 100644 qmq-metrics-prometheus/src/main/java/qunar/tc/qmq/metrics/prometheus/PrometheusQmqTimer.java create mode 100644 qmq-metrics-prometheus/src/main/resources/META-INF/services/qunar.tc.qmq.metrics.QmqMetricRegistry create mode 100644 qmq-remoting/pom.xml create mode 100644 qmq-remoting/src/main/java/qunar/tc/qmq/netty/DecodeHandler.java create mode 100644 qmq-remoting/src/main/java/qunar/tc/qmq/netty/EncodeHandler.java create mode 100644 qmq-remoting/src/main/java/qunar/tc/qmq/netty/NettyClientConfig.java create mode 100644 qmq-remoting/src/main/java/qunar/tc/qmq/netty/client/AbstractNettyClient.java create mode 100644 qmq-remoting/src/main/java/qunar/tc/qmq/netty/client/ChannelWrapper.java create mode 100644 qmq-remoting/src/main/java/qunar/tc/qmq/netty/client/HttpResponseCallback.java create mode 100644 qmq-remoting/src/main/java/qunar/tc/qmq/netty/client/NettyClient.java create mode 100644 qmq-remoting/src/main/java/qunar/tc/qmq/netty/client/NettyClientHandler.java create mode 100644 qmq-remoting/src/main/java/qunar/tc/qmq/netty/client/NettyConnectManageHandler.java create mode 100644 qmq-remoting/src/main/java/qunar/tc/qmq/netty/client/Response.java create mode 100644 qmq-remoting/src/main/java/qunar/tc/qmq/netty/client/ResponseFuture.java create mode 100644 qmq-remoting/src/main/java/qunar/tc/qmq/netty/exception/BrokerRejectException.java create mode 100644 qmq-remoting/src/main/java/qunar/tc/qmq/netty/exception/ClientSendException.java create mode 100644 qmq-remoting/src/main/java/qunar/tc/qmq/netty/exception/EncodeException.java create mode 100644 qmq-remoting/src/main/java/qunar/tc/qmq/netty/exception/RemoteBusyException.java create mode 100644 qmq-remoting/src/main/java/qunar/tc/qmq/netty/exception/RemoteException.java create mode 100644 qmq-remoting/src/main/java/qunar/tc/qmq/netty/exception/RemoteResponseUnreadableException.java create mode 100644 qmq-remoting/src/main/java/qunar/tc/qmq/netty/exception/RemoteTimeoutException.java create mode 100644 qmq-remoting/src/main/java/qunar/tc/qmq/netty/exception/SubjectNotAssignedException.java create mode 100644 qmq-remoting/src/main/java/qunar/tc/qmq/protocol/CommandCode.java create mode 100644 qmq-remoting/src/main/java/qunar/tc/qmq/protocol/Datagram.java create mode 100644 qmq-remoting/src/main/java/qunar/tc/qmq/protocol/MessagesPayloadHolder.java create mode 100644 qmq-remoting/src/main/java/qunar/tc/qmq/protocol/PayloadHolder.java create mode 100644 qmq-remoting/src/main/java/qunar/tc/qmq/protocol/QMQSerializer.java create mode 100644 qmq-remoting/src/main/java/qunar/tc/qmq/protocol/RemotingCommand.java create mode 100644 qmq-remoting/src/main/java/qunar/tc/qmq/protocol/RemotingCommandType.java create mode 100644 qmq-remoting/src/main/java/qunar/tc/qmq/protocol/RemotingHeader.java create mode 100644 qmq-remoting/src/main/java/qunar/tc/qmq/protocol/consumer/AckRequest.java create mode 100644 qmq-remoting/src/main/java/qunar/tc/qmq/protocol/consumer/AckRequestPayloadHolder.java create mode 100644 qmq-remoting/src/main/java/qunar/tc/qmq/protocol/consumer/MetaInfoRequest.java create mode 100644 qmq-remoting/src/main/java/qunar/tc/qmq/protocol/consumer/MetaInfoRequestPayloadHolder.java create mode 100644 qmq-remoting/src/main/java/qunar/tc/qmq/protocol/consumer/MetaInfoResponse.java create mode 100644 qmq-remoting/src/main/java/qunar/tc/qmq/protocol/consumer/PullRequest.java create mode 100644 qmq-remoting/src/main/java/qunar/tc/qmq/protocol/consumer/PullRequestPayloadHolder.java create mode 100644 qmq-remoting/src/main/java/qunar/tc/qmq/protocol/producer/MessageProducerCode.java create mode 100644 qmq-remoting/src/main/java/qunar/tc/qmq/protocol/producer/SendResult.java create mode 100644 qmq-remoting/src/main/java/qunar/tc/qmq/util/ChannelUtil.java create mode 100644 qmq-remoting/src/main/java/qunar/tc/qmq/util/RemoteHelper.java create mode 100644 qmq-remoting/src/main/java/qunar/tc/qmq/util/RemotingBuilder.java create mode 100644 qmq-server-common/pom.xml create mode 100644 qmq-server-common/src/main/java/qunar/tc/qmq/base/ActionLogOffsetRequest.java create mode 100644 qmq-server-common/src/main/java/qunar/tc/qmq/base/ActionLogOffsetResponse.java create mode 100644 qmq-server-common/src/main/java/qunar/tc/qmq/base/ConsumeManageRequest.java create mode 100644 qmq-server-common/src/main/java/qunar/tc/qmq/base/ConsumerLag.java create mode 100644 qmq-server-common/src/main/java/qunar/tc/qmq/base/QueryConsumerLagRequest.java create mode 100644 qmq-server-common/src/main/java/qunar/tc/qmq/base/SyncRequest.java create mode 100644 qmq-server-common/src/main/java/qunar/tc/qmq/concurrent/ActorSystem.java create mode 100644 qmq-server-common/src/main/java/qunar/tc/qmq/configuration/BrokerConfig.java create mode 100644 qmq-server-common/src/main/java/qunar/tc/qmq/constants/BrokerConstants.java create mode 100644 qmq-server-common/src/main/java/qunar/tc/qmq/meta/BrokerAcquireMetaRequest.java create mode 100644 qmq-server-common/src/main/java/qunar/tc/qmq/meta/BrokerAcquireMetaRequestSerializer.java create mode 100644 qmq-server-common/src/main/java/qunar/tc/qmq/meta/BrokerAcquireMetaResponse.java create mode 100644 qmq-server-common/src/main/java/qunar/tc/qmq/meta/BrokerAcquireMetaResponseSerializer.java create mode 100644 qmq-server-common/src/main/java/qunar/tc/qmq/meta/BrokerRegisterRequest.java create mode 100644 qmq-server-common/src/main/java/qunar/tc/qmq/meta/BrokerRegisterRequestSerializer.java create mode 100644 qmq-server-common/src/main/java/qunar/tc/qmq/meta/BrokerRegisterResponse.java create mode 100644 qmq-server-common/src/main/java/qunar/tc/qmq/meta/BrokerRegisterService.java create mode 100644 qmq-server-common/src/main/java/qunar/tc/qmq/meta/BrokerRequestType.java create mode 100644 qmq-server-common/src/main/java/qunar/tc/qmq/meta/BrokerRole.java create mode 100644 qmq-server-common/src/main/java/qunar/tc/qmq/monitor/QMon.java create mode 100644 qmq-server-common/src/main/java/qunar/tc/qmq/netty/ConnectionEventHandler.java create mode 100644 qmq-server-common/src/main/java/qunar/tc/qmq/netty/ConnectionHandler.java create mode 100644 qmq-server-common/src/main/java/qunar/tc/qmq/netty/DefaultConnectionEventHandler.java create mode 100644 qmq-server-common/src/main/java/qunar/tc/qmq/netty/NettyRequestExecutor.java create mode 100644 qmq-server-common/src/main/java/qunar/tc/qmq/netty/NettyRequestProcessor.java create mode 100644 qmq-server-common/src/main/java/qunar/tc/qmq/netty/NettyServer.java create mode 100644 qmq-server-common/src/main/java/qunar/tc/qmq/netty/NettyServerHandler.java create mode 100644 qmq-server-common/src/main/java/qunar/tc/qmq/service/HeartbeatManager.java create mode 100644 qmq-server/pom.xml create mode 100644 qmq-server/src/main/java/qunar/tc/qmq/base/ConsumerGroup.java create mode 100644 qmq-server/src/main/java/qunar/tc/qmq/base/ConsumerSequence.java create mode 100644 qmq-server/src/main/java/qunar/tc/qmq/base/PullMessageResult.java create mode 100644 qmq-server/src/main/java/qunar/tc/qmq/base/ReceiveResult.java create mode 100644 qmq-server/src/main/java/qunar/tc/qmq/base/ReceivingMessage.java create mode 100644 qmq-server/src/main/java/qunar/tc/qmq/base/WritePutActionResult.java create mode 100644 qmq-server/src/main/java/qunar/tc/qmq/consumer/ConsumerSequenceManager.java create mode 100644 qmq-server/src/main/java/qunar/tc/qmq/consumer/OfflineActionHandler.java create mode 100644 qmq-server/src/main/java/qunar/tc/qmq/consumer/OfflineTask.java create mode 100644 qmq-server/src/main/java/qunar/tc/qmq/consumer/RetryTask.java create mode 100644 qmq-server/src/main/java/qunar/tc/qmq/consumer/Subscriber.java create mode 100644 qmq-server/src/main/java/qunar/tc/qmq/consumer/SubscriberStatusChecker.java create mode 100644 qmq-server/src/main/java/qunar/tc/qmq/container/Bootstrap.java create mode 100644 qmq-server/src/main/java/qunar/tc/qmq/lag/ConsumerLagService.java create mode 100644 qmq-server/src/main/java/qunar/tc/qmq/processor/AbstractRequestProcessor.java create mode 100644 qmq-server/src/main/java/qunar/tc/qmq/processor/AckMessageProcessor.java create mode 100644 qmq-server/src/main/java/qunar/tc/qmq/processor/AckMessageWorker.java create mode 100644 qmq-server/src/main/java/qunar/tc/qmq/processor/BrokerConnectionEventHandler.java create mode 100644 qmq-server/src/main/java/qunar/tc/qmq/processor/GetQueueCountProcessor.java create mode 100644 qmq-server/src/main/java/qunar/tc/qmq/processor/PullMessageProcessor.java create mode 100644 qmq-server/src/main/java/qunar/tc/qmq/processor/PullMessageWorker.java create mode 100644 qmq-server/src/main/java/qunar/tc/qmq/processor/SendMessageProcessor.java create mode 100644 qmq-server/src/main/java/qunar/tc/qmq/processor/SendMessageWorker.java create mode 100644 qmq-server/src/main/java/qunar/tc/qmq/processor/filters/Invoker.java create mode 100644 qmq-server/src/main/java/qunar/tc/qmq/processor/filters/ReceiveFilter.java create mode 100644 qmq-server/src/main/java/qunar/tc/qmq/processor/filters/ReceiveFilterChain.java create mode 100644 qmq-server/src/main/java/qunar/tc/qmq/processor/filters/ValidateFilter.java create mode 100644 qmq-server/src/main/java/qunar/tc/qmq/startup/ServerWrapper.java create mode 100644 qmq-server/src/main/java/qunar/tc/qmq/stats/BrokerStats.java create mode 100644 qmq-server/src/main/java/qunar/tc/qmq/stats/PerMinuteDeltaCounter.java create mode 100644 qmq-server/src/main/java/qunar/tc/qmq/store/MessageStoreWrapper.java create mode 100644 qmq-server/src/main/java/qunar/tc/qmq/store/Tags.java create mode 100644 qmq-server/src/main/java/qunar/tc/qmq/sync/master/AbstractLogSyncWorker.java create mode 100644 qmq-server/src/main/java/qunar/tc/qmq/sync/master/ActionLogSyncWorker.java create mode 100644 qmq-server/src/main/java/qunar/tc/qmq/sync/master/HeartbeatSyncWorker.java create mode 100644 qmq-server/src/main/java/qunar/tc/qmq/sync/master/MasterSyncNettyServer.java create mode 100644 qmq-server/src/main/java/qunar/tc/qmq/sync/master/MessageLogSyncWorker.java create mode 100644 qmq-server/src/main/java/qunar/tc/qmq/sync/master/SyncCheckpointProcessor.java create mode 100644 qmq-server/src/main/java/qunar/tc/qmq/sync/master/SyncLogProcessor.java create mode 100644 qmq-server/src/main/java/qunar/tc/qmq/sync/master/SyncProcessor.java create mode 100644 qmq-server/src/main/java/qunar/tc/qmq/sync/master/SyncRequestEntry.java create mode 100644 qmq-server/src/main/java/qunar/tc/qmq/utils/ConsumerGroupUtils.java create mode 100644 qmq-server/src/main/java/qunar/tc/qmq/utils/HeaderSerializer.java create mode 100644 qmq-server/src/main/java/qunar/tc/qmq/utils/ServerTimerUtil.java create mode 100644 qmq-store/pom.xml create mode 100644 qmq-store/src/main/c/MmapUtil.c create mode 100644 qmq-store/src/main/c/MmapUtil.h create mode 100644 qmq-store/src/main/java/qunar/tc/qmq/store/Action.java create mode 100644 qmq-store/src/main/java/qunar/tc/qmq/store/ActionCheckpoint.java create mode 100644 qmq-store/src/main/java/qunar/tc/qmq/store/ActionCheckpointSerde.java create mode 100644 qmq-store/src/main/java/qunar/tc/qmq/store/ActionLog.java create mode 100644 qmq-store/src/main/java/qunar/tc/qmq/store/ActionLogIterateService.java create mode 100644 qmq-store/src/main/java/qunar/tc/qmq/store/ActionLogVisitor.java create mode 100644 qmq-store/src/main/java/qunar/tc/qmq/store/ActionReaderWriter.java create mode 100644 qmq-store/src/main/java/qunar/tc/qmq/store/ActionType.java create mode 100644 qmq-store/src/main/java/qunar/tc/qmq/store/AppendMessageResult.java create mode 100644 qmq-store/src/main/java/qunar/tc/qmq/store/AppendMessageStatus.java create mode 100644 qmq-store/src/main/java/qunar/tc/qmq/store/CheckpointLoader.java create mode 100644 qmq-store/src/main/java/qunar/tc/qmq/store/CheckpointManager.java create mode 100644 qmq-store/src/main/java/qunar/tc/qmq/store/CheckpointStore.java create mode 100644 qmq-store/src/main/java/qunar/tc/qmq/store/ConsumeQueue.java create mode 100644 qmq-store/src/main/java/qunar/tc/qmq/store/ConsumeQueueManager.java create mode 100644 qmq-store/src/main/java/qunar/tc/qmq/store/ConsumerGroupProgress.java create mode 100644 qmq-store/src/main/java/qunar/tc/qmq/store/ConsumerLog.java create mode 100644 qmq-store/src/main/java/qunar/tc/qmq/store/ConsumerLogEntry.java create mode 100644 qmq-store/src/main/java/qunar/tc/qmq/store/ConsumerLogFlusher.java create mode 100644 qmq-store/src/main/java/qunar/tc/qmq/store/ConsumerLogManager.java create mode 100644 qmq-store/src/main/java/qunar/tc/qmq/store/ConsumerLogWroteEvent.java create mode 100644 qmq-store/src/main/java/qunar/tc/qmq/store/ConsumerProgress.java create mode 100644 qmq-store/src/main/java/qunar/tc/qmq/store/DataTransfer.java create mode 100644 qmq-store/src/main/java/qunar/tc/qmq/store/DefaultStorage.java create mode 100644 qmq-store/src/main/java/qunar/tc/qmq/store/FlushHook.java create mode 100644 qmq-store/src/main/java/qunar/tc/qmq/store/GetMessageResult.java create mode 100644 qmq-store/src/main/java/qunar/tc/qmq/store/GetMessageStatus.java create mode 100644 qmq-store/src/main/java/qunar/tc/qmq/store/GroupAndSubject.java create mode 100644 qmq-store/src/main/java/qunar/tc/qmq/store/LogManager.java create mode 100644 qmq-store/src/main/java/qunar/tc/qmq/store/LogSegment.java create mode 100644 qmq-store/src/main/java/qunar/tc/qmq/store/LogSegmentValidator.java create mode 100644 qmq-store/src/main/java/qunar/tc/qmq/store/MagicCode.java create mode 100644 qmq-store/src/main/java/qunar/tc/qmq/store/MagicCodeSupport.java create mode 100644 qmq-store/src/main/java/qunar/tc/qmq/store/MaxAckedPullLogSequence.java create mode 100644 qmq-store/src/main/java/qunar/tc/qmq/store/MaxPulledMessageSequence.java create mode 100644 qmq-store/src/main/java/qunar/tc/qmq/store/MessageAppender.java create mode 100644 qmq-store/src/main/java/qunar/tc/qmq/store/MessageCheckpoint.java create mode 100644 qmq-store/src/main/java/qunar/tc/qmq/store/MessageCheckpointSerde.java create mode 100644 qmq-store/src/main/java/qunar/tc/qmq/store/MessageFilter.java create mode 100644 qmq-store/src/main/java/qunar/tc/qmq/store/MessageLog.java create mode 100644 qmq-store/src/main/java/qunar/tc/qmq/store/MessageLogIterateService.java create mode 100644 qmq-store/src/main/java/qunar/tc/qmq/store/MessageLogMeta.java create mode 100644 qmq-store/src/main/java/qunar/tc/qmq/store/MessageLogMetaVisitor.java create mode 100644 qmq-store/src/main/java/qunar/tc/qmq/store/MessageSequence.java create mode 100644 qmq-store/src/main/java/qunar/tc/qmq/store/MmapUtil.java create mode 100644 qmq-store/src/main/java/qunar/tc/qmq/store/OffsetBound.java create mode 100644 qmq-store/src/main/java/qunar/tc/qmq/store/OffsetRange.java create mode 100644 qmq-store/src/main/java/qunar/tc/qmq/store/PeriodicFlushService.java create mode 100644 qmq-store/src/main/java/qunar/tc/qmq/store/PullLog.java create mode 100644 qmq-store/src/main/java/qunar/tc/qmq/store/PullLogFlusher.java create mode 100644 qmq-store/src/main/java/qunar/tc/qmq/store/PullLogManager.java create mode 100644 qmq-store/src/main/java/qunar/tc/qmq/store/PullLogMessage.java create mode 100644 qmq-store/src/main/java/qunar/tc/qmq/store/PutMessageResult.java create mode 100644 qmq-store/src/main/java/qunar/tc/qmq/store/PutMessageStatus.java create mode 100644 qmq-store/src/main/java/qunar/tc/qmq/store/ReferenceObject.java create mode 100644 qmq-store/src/main/java/qunar/tc/qmq/store/SegmentBuffer.java create mode 100644 qmq-store/src/main/java/qunar/tc/qmq/store/Serde.java create mode 100644 qmq-store/src/main/java/qunar/tc/qmq/store/Snapshot.java create mode 100644 qmq-store/src/main/java/qunar/tc/qmq/store/SnapshotStore.java create mode 100644 qmq-store/src/main/java/qunar/tc/qmq/store/Storage.java create mode 100644 qmq-store/src/main/java/qunar/tc/qmq/store/StorageConfig.java create mode 100644 qmq-store/src/main/java/qunar/tc/qmq/store/StorageConfigImpl.java create mode 100644 qmq-store/src/main/java/qunar/tc/qmq/store/StoreUtils.java create mode 100644 qmq-store/src/main/java/qunar/tc/qmq/store/action/ActionEvent.java create mode 100644 qmq-store/src/main/java/qunar/tc/qmq/store/action/ForeverOfflineAction.java create mode 100644 qmq-store/src/main/java/qunar/tc/qmq/store/action/ForeverOfflineActionReaderWriter.java create mode 100644 qmq-store/src/main/java/qunar/tc/qmq/store/action/MaxSequencesUpdater.java create mode 100644 qmq-store/src/main/java/qunar/tc/qmq/store/action/PullAction.java create mode 100644 qmq-store/src/main/java/qunar/tc/qmq/store/action/PullActionReaderWriter.java create mode 100644 qmq-store/src/main/java/qunar/tc/qmq/store/action/PullLogBuilder.java create mode 100644 qmq-store/src/main/java/qunar/tc/qmq/store/action/RangeAckAction.java create mode 100644 qmq-store/src/main/java/qunar/tc/qmq/store/action/RangeAckActionReaderWriter.java create mode 100644 qmq-store/src/main/java/qunar/tc/qmq/store/event/FixedExecOrderEventBus.java create mode 100644 qmq-store/src/main/resources/META-INF/native/libmmaputil.so create mode 100644 qmq-sync/pom.xml create mode 100644 qmq-sync/src/main/java/qunar/tc/qmq/sync/AbstractSyncLogProcessor.java create mode 100644 qmq-sync/src/main/java/qunar/tc/qmq/sync/DelaySyncRequest.java create mode 100644 qmq-sync/src/main/java/qunar/tc/qmq/sync/HeartbeatProcessor.java create mode 100644 qmq-sync/src/main/java/qunar/tc/qmq/sync/MasterSlaveSyncManager.java create mode 100644 qmq-sync/src/main/java/qunar/tc/qmq/sync/SlaveSyncClient.java create mode 100644 qmq-sync/src/main/java/qunar/tc/qmq/sync/SlaveSyncSender.java create mode 100644 qmq-sync/src/main/java/qunar/tc/qmq/sync/SyncActionLogProcessor.java create mode 100644 qmq-sync/src/main/java/qunar/tc/qmq/sync/SyncLogProcessor.java create mode 100644 qmq-sync/src/main/java/qunar/tc/qmq/sync/SyncMessageLogProcessor.java create mode 100644 qmq-sync/src/main/java/qunar/tc/qmq/sync/SyncType.java create mode 100644 qmq-tools/pom.xml create mode 100644 qmq-tools/src/main/java/qunar/tc/qmq/tools/MetaManagementService.java create mode 100644 qmq-tools/src/main/java/qunar/tc/qmq/tools/Tools.java create mode 100644 qmq-tools/src/main/java/qunar/tc/qmq/tools/command/AddBrokerCommand.java create mode 100644 qmq-tools/src/main/java/qunar/tc/qmq/tools/command/AddNewSubjectCommand.java create mode 100644 qmq-tools/src/main/java/qunar/tc/qmq/tools/command/AddSubjectBrokerGroupCommand.java create mode 100644 qmq-tools/src/main/java/qunar/tc/qmq/tools/command/ExtendSubjectRouteCommand.java create mode 100644 qmq-tools/src/main/java/qunar/tc/qmq/tools/command/ListBrokerGroupsCommand.java create mode 100644 qmq-tools/src/main/java/qunar/tc/qmq/tools/command/ListBrokersCommand.java create mode 100644 qmq-tools/src/main/java/qunar/tc/qmq/tools/command/ListSubjectRoutesCommand.java create mode 100644 qmq-tools/src/main/java/qunar/tc/qmq/tools/command/RemoveSubjectBrokerGroupCommand.java create mode 100644 qmq-tools/src/main/java/qunar/tc/qmq/tools/command/ReplaceBrokerCommand.java diff --git a/.gitignore b/.gitignore new file mode 100644 index 00000000..24bb47b3 --- /dev/null +++ b/.gitignore @@ -0,0 +1,37 @@ +# kdiff3 ignore +*.orig + +# maven ignore +target/ + +# eclipse ignore +.settings/ +.project +.classpath + +# idea ignore +.idea/ +*.ipr +*.iml +*.iws + +# temp ignore +*.log +*.cache +*.diff +*.patch +*.tmp + +# system ignore +.DS_Store +Thumbs.db + +# package ignore (optional) +# *.jar +# *.war +# *.zip +# *.tar +# *.tar.gz +catalina.base_IS_UNDEFINED/ + +qconfig_test \ No newline at end of file diff --git a/Makefile b/Makefile new file mode 100644 index 00000000..8451e027 --- /dev/null +++ b/Makefile @@ -0,0 +1,2 @@ +package: + mvn -U clean package -Dmaven.test.skip=true -DskipTests -am -pl qmq-dist diff --git a/README.md b/README.md new file mode 100644 index 00000000..944c5c29 --- /dev/null +++ b/README.md @@ -0,0 +1,43 @@ +# QMQ + +QMQ是去哪儿网内部广泛使用的消息中间件,自2012年诞生以来在去哪儿网所有业务场景中广泛的应用,包括跟交易息息相关的订单场景; +也包括报价搜索等高吞吐量场景。目前在公司内部日常消息qps在60W左右,生产上承载将近4W+消息topic,消息的端到端延迟可以控制在10ms以内。 + +主要提供以下特性: +* 异步实时消息 +* 延迟/定时消息 +* 基于Tag的服务端过滤 +* Consumer端幂等处理支持 +* Consumer端filter +* 死信消息 +* 结合Spring annotation使用的简单API +* 提供丰富的监控指标 +* 接入OpenTracing +* 分布式事务(即将开源) +* 消息投递轨迹(即将开源) +* 历史消息的自动备份(即将开源) + +# 快速开始 +你可以通过[设计背景](docs/cn/design.md)了解设计QMQ的初衷和她与其他消息队列的不同。 +阅读[架构概览](docs/cn/arch.md)了解QMQ的存储模型 + +## 文档 +* [快速入门](docs/cn/quickstart.md) +* [安装](docs/cn/install.md) +* [设计背景](docs/cn/design.md) +* [架构概览](docs/cn/arch.md) +* [代码模块介绍](docs/cn/code.md) +* [高可用](docs/cn/ha.md) +* [监控](docs/cn/monitor.md) +* [Trace](docs/cn/trace.md) +* [发送消息](docs/cn/producer.md) +* [消费消息](docs/cn/consumer.md) +* [延时/定时消息](docs/cn/delay.md) +* [服务端tag过滤](docs/cn/tag.md) +* [开源协议](docs/cn/opensource.md) +* [技术支持](docs/cn/support.md) + +# 技术支持 + +### QQ群 +![QQ](docs/images/support1.png) \ No newline at end of file diff --git a/docs/cn/arch.md b/docs/cn/arch.md new file mode 100644 index 00000000..3e881519 --- /dev/null +++ b/docs/cn/arch.md @@ -0,0 +1,67 @@ +[上一页](design.md) +[回目录](../../readme.md) +[下一页](code.md) + +# 架构概览 +下图是QMQ中各组件及其交互图: +* meta server提供集群管理和集群发现的作用 +* server 提供实时消息服务 +* delay server 提供延时/定时消息服务,延时消息先在delay server排队,时间到之后再发送给server +* producer 消息生产者 +* consumer 消息消费者 + +![架构图](../images/arch1.png) + +根据图中的编号描述一下其交互过程 +1. delay server 向meta server注册 +2. 实时server 向meta server注册 +3. producer在发送消息前需要询问meta server获取server list +4. meta server返回server list给producer(根据producer请求的消息类型返回不同的server list) +5. producer发送延时/定时消息 +6. 延时时间已到,delay server将消息投递给实时server +7. producer发送实时消息 +8. consumer需要拉取消息,在拉取之前向meta server获取server list(只会获取实时server的list) +9. meta server返回server list给consumer +10. consumer向实时server发起pull请求 +11. 实时server将消息返回给consumer + +下面分别对实时消息Server和延时/定时消息Server的存储模型进行描述 + +## 实时消息 +在设计背景里,已经描述了QMQ没有采用基于partition存储模型,但是在学习Kafka和RocketMQ的存储实现方式后,有很多地方是值得借鉴的: +* 顺序append文件,提供很好的性能 +* 顺序消费文件,使用offset表示消费进度,成本极低 +* 将所有subject的消息合并在一起,减少parition数量,可以提供更多的subject(RocketMQ) + +在演化QMQ的存储模型时,觉得这几点是非常重要的。那如何在不实用基于partition的情况下,又能得到这些特性呢?正所谓有前辈大师说:计算机中所有问题都可以通过添加一个中间层来解决,一个中间层解决不了那就添加两个。 + +我们通过添加一层拉取的log(pull log)来动态映射consumer与partition的逻辑关系,这样不仅解决了consumer的动态扩容缩容问题,还可以继续使用一个offset表示消费进度。 + +下图是QMQ的存储模型 + +![img](../images/arch3.png) + +先解释一下上图中的数字的意义。上图中方框上方的数字,表示该方框在自己log中的偏移,而方框内的数字是该项的内容。比如message log方框上方的数字:3,6,9几表示这几条消息在message log中的偏移。而consume log中方框内的数字3,6,9,20正对应着messzge log的偏移,表示这几个位置上的消息都是subject1的消息,consume log方框上方的1,2,3,4表示这几个方框在consume log中的逻辑偏移。下面的pull log方框内的内容对应着consume log的逻辑偏移,而pull log方框外的数字表示pull log的逻辑偏移。 + +在实时Server存储模型中有三种重要的log: +* message log 所有subject的消息进入该log,消息的主存储 +* consume log consume log存储的是message log的索引信息 +* pull log 每个consumer拉取消息的时候会产生pull log,pull log记录的是拉取的消息在consume log中的sequence + +那么消费者就可以使用pull log上的sequence来表示消费进度 + +## 延时/定时消息 +QMQ提供任意时间的延时/定时消息,你可以指定消息在未来两年内任意时间内投递。比起RocketMQ提供的多个不同延时level的延时消息,QMQ的延时消息更加灵活。比如在OTA场景中,客人经常是预订未来某个时刻的酒店或者机票,这个时间是不固定的,我们无法使用几个固定的延时level来实现这个场景。 + +QMQ的延时/定时消息使用的是两层HashWheelTimer来实现的。第一层位于磁盘上,每个小时为一个刻度,每个刻度会生成一个日志文件,因为QMQ支持两年内的延迟消息,则最多会生成 2 * 366 * 24 = 17568 个文件。第二层在内存中,当消息的投递时间即将到来的时候,会将这个小时的消息索引从磁盘文件加载到内存中的HashWheelTimer上。 + +![img](../images/arch4.png) + +在延时/定时消息里也存在三种log: +* message log 和实时消息里的message log类似,收到消息后append到该log +* schedule log 按照投递时间组织,每个小时一个。该log是回放message log后根据延时时间放置对应的log上,这是上面描述的两层HashWheelTimer的第一层,位于磁盘上 +* dispatch log 延时/定时消息投递后写入,主要用于在应用重启后能够确定哪些消息已经投递 + +[上一页](design.md) +[回目录](../../readme.md) +[下一页](code.md) \ No newline at end of file diff --git a/docs/cn/code.md b/docs/cn/code.md new file mode 100644 index 00000000..03586b17 --- /dev/null +++ b/docs/cn/code.md @@ -0,0 +1,36 @@ +[上一页](arch.md) +[回目录](../../readme.md) +[下一页](ha.md) + +# 代码模块介绍 + +### qmq-api +暴露给用户的一些interface + +### qmq-common +一些公用的类,所有其他模块都可能引用 + +### qmq-server-common +公用的类,只有server side应用引用,不暴露给client side + +### qmq-server +实时消息服务 + +### qmq-delay-server +延时/定时消息服务 + +### qmq-store +存储 + +### qmq-remoting +网络相关 + +### qmq-client +客户端逻辑 + +### qmq-metrics-prometheus +提供的prometheus监控接入 + +[上一页](arch.md) +[回目录](../../readme.md) +[下一页](ha.md) \ No newline at end of file diff --git a/docs/cn/consumer.md b/docs/cn/consumer.md new file mode 100644 index 00000000..840a9bbe --- /dev/null +++ b/docs/cn/consumer.md @@ -0,0 +1,122 @@ +[上一页](producer.md) +[回目录](../../readme.md) +[下一页](delay.md) + + +# 消费消息(consumer) + +## 与Spring结合 + +QMQ除了提供使用API来消费消息的方式外,还提供了跟Spring结合的基于annotation的API,我们更推荐使用这种方式。QMQ已经与SpringBoot进行了集成,如果项目使用SpringBoot则只需要引入qmq-client.jar就可以直接使用annotation的API,如果使用传统Spring的话则需要在Spring的xml里进行如下配置: +```xml + + + + + + + + +``` + +使用下面的代码就可以订阅消息了,是不是非常简单。 +```java +@QmqConsumer(subject = "your subject", consumerGroup = "group", executor = "your executor bean name") +public void onMessage(Message message){ + //process your message + String value = message.getStringProperty("key"); +} +``` +使用上面的方式订阅消息时,如果QmqConsumer标记的onMessage方法抛出异常,则该方法被认为是消费失败,消费失败的消息会再次消费,默认再次消费的间隔是5秒钟,这个可以进行配置。这里需要注意的是,如果有些通过重试也无法消除的异常,请将其在onMessage方法里捕获,而通过重试可以恢复的异常才抛出。 + +### 仅消费一次 +有些消息的可靠性可能要求不高,不管是消费成功还是失败,仅仅消费一次即可,不期望重试,那么可以设置仅消费一次 +```java +@QmqConsumer(subject = "your subject", consumerGroup="group", consumeMostOnce = true, executor = "your executor bean name") +public void onMessage(Message message){ + //process your message + String value = message.getStringProperty("key"); +} +``` + +### 广播消息 +有这样的场景,我们每台机器都维护进程内内存,当数据有变更的时候,变更方会发送变更消息触发缓存更新,那么这个时候我们期望消费者每台机器都收到消息,这就是广播消息的场景了。 +```java +@QmqConsumer(subject = "your subject", consumerGroup="group", isBroadcast = true, executor = "your executor bean name") +public void onMessage(Message message){ + //update local cache +} +``` + +### 消费端过滤器(filter) +可以将一些公共逻辑放在filter里,这样可以将filter配置在所有消费者上。比如在QMQ里内置了opentracing就是通过filter实现的,不过这个filter是内置的,不需要额外配置。 +```java + +@Compoent +public class LogFilter implements Filter { + + //在处理消息之前执行 + public boolean preOnMessage(Message message, Map filterContext){ + + } + + //在处理消息之后执行 + public void postOnMessage(Message message, Throwable e, Map filterContext){ + + } +} + +@QmqConsumer(subject = "your subject", consumerGroup="group", filters = {"logFilter"}, executor = "your executor bean name") +public void onMessage(Message message){ + //update local cache +} +``` + +## 非Spring API +如果在非Spring环境中使用QMQ那就需要直接使用API了。QMQ提供了两种API:Listener和纯Pull。 + +### Listener + +Listener的方式与@QmqConsumer提供的功能基本类似 + +```java +//推荐一个应用里只创建一个实例 +MessageConsumerProvider consumer = new MessageConsumerProvider(); +consumer.init(); + +consumer.addListener("your subject", "group", (m) -> { + //process message +}, new ThreadPoolExecutor(2,2,)); +``` + +### Pull API + +Pull API是最基础的API,需要考虑更多情况,如无必要,我们推荐使用annotation或者Listener的方式。 + +```java +//推荐一个应用里只创建一个实例 +MessageConsumerProvider consumer = new MessageConsumerProvider(); +consumer.init(); + +PullConsumer pullConsumer = consumer.getOrCreatePullConsumer("your subject", "group", false); +List messages = pullConsumer.pull(100, 1000); +for(Message message : messages){ + //process message + + //对于pull的使用方式,pull到的每一条消息都必须ack,如果处理成功ack的第二个参数为null + message.ack(1000, null); + + //处理失败,则ack的第二个参数传入Throwable对象 + //message.ack(1000, new Exception("消费失败")); +} +``` + +[上一页](producer.md) +[回目录](../../readme.md) +[下一页](delay.md) \ No newline at end of file diff --git a/docs/cn/contributing.md b/docs/cn/contributing.md new file mode 100644 index 00000000..38a5da75 --- /dev/null +++ b/docs/cn/contributing.md @@ -0,0 +1,5 @@ +如果你遇到问题或需要新功能,欢迎[创建issue] + +如果你可以解决某个[issue], 欢迎发送PR + +如果你觉得该项目对你有帮助,也请不吝Star \ No newline at end of file diff --git a/docs/cn/delay.md b/docs/cn/delay.md new file mode 100644 index 00000000..2556e0d8 --- /dev/null +++ b/docs/cn/delay.md @@ -0,0 +1,46 @@ +[上一页](consumer.md) +[回目录](../../readme.md) +[下一页](tag.md) + +# 延时/定时消息 + +延时/定时消息是指生产者(producer)发送消息到server后,server并不将消息立即发送给消费者(consumer),而是在producer指定的时间之后送达。比如在电商交易中,经常有这样的场景:下单后如果半个小时内没有支付,自动将订单取消。那么如果不使用延时/定时消息,则一般的做法是使用定时任务定期扫描订单状态表,如果半个小时后订单状态还未支付,则将订单取消。而使用延时/定时消息实现起来则比较优雅:用户下单后,发送一个延时消息,指定半个小时后消息送达,那么消费者在半个小时后收到消息就查询消息状态,如果这个时候订单是未支付状态,则取消订单。 + +延时/定时消息的消费与实时消息一致,请参照[消费消息](consumer.md) + +注意: 延时/定时消息使用的时间都是指delay server服务器的时间,所以请确保delay server的服务器时间偏差不要太大。另外,延时/定时消息的精度在1秒左右,在业务设计时应该考虑到这一点,比如不要期望能达到延时100ms的效果。 + +## 发送延时消息 + +延时消息是指消息在当前时间之后一段时间后发送 + +```java +Message message = producer.generateMessage("your subject"); +message.setProperty("key", "value"); + +//指定消息延时30分钟 +message.setDelayTime(30, TimeUnit.MINUTES); + +//发送消息 +producer.sendMessage(message); +``` + +## 发送定时消息 + +定时消息是指指定消息的发送时间。需要注意的是如果指定的发送时间小于或等于当前时间,消息是会立即发送的 + +```java +Message message = producer.generateMessage("your subject"); +message.setProperty("key", "value"); + +Date deadline = DateUtils.addMinutes(new Date(), 30); +//指定发送时间 +message.setDelayTime(deadline); + +//发送消息 +producer.sendMessage(message); +``` + +[上一页](consumer.md) +[回目录](../../readme.md) +[下一页](tag.md) \ No newline at end of file diff --git a/docs/cn/design.md b/docs/cn/design.md new file mode 100644 index 00000000..bb2def75 --- /dev/null +++ b/docs/cn/design.md @@ -0,0 +1,38 @@ +[上一页](install.md) +[回目录](../../readme.md) +[下一页](arch.md) + + +# 设计背景 + +现在市面上已经有很多消息中间件了(ActiveMQ, RabbitMQ, Kafka, RocketMQ),那么为什么我们还要造另外一个轮子呢?首先QMQ是2012年就开始开发的,在这个时期其实消息中间件并没有太多的选择,那个时候Kafka还不太成熟,而RocketMQ也没有出现,大部分公司都会采用ActiveMQ或RabbitMQ。 +首先RabbitMQ的开发语言是erlang,这是一门略小众的语言,我们担心无法完全掌控,所以没有选择。而ActiveMQ其实在公司内部已有很长一段时间使用历史,但是ActiveMQ太过于复杂,在使用过程中经常出现消息丢失或者整个进程hang住的情况,并且难以定位。 + +当然,这都是QMQ诞生的历史背景,那么放在今天QMQ还有开源的意义吗?我们比较了市面上现有的消息中间件,仍然认为QMQ有自己的特色,而这些也是我们在过去几年运维消息中间件中觉得必须提供的和难以舍弃的。 + +我们都知道Kafka和RocketMQ都是基于partition的存储模型,也就是每个subject分为一个或多个partition,而Server收到消息后将其分发到某个partition上,而Consumer消费的时候是与partition对应的。比如,我们某个subject a分配了3个partition(p1, p2, p3),有3个消费者(c1, c2, c3)消费该消息,则会建立c1 - p1, c2 - p2, c3 - p3这样的消费关系。 + +![img](../images/design1.png) + +那么如果我们的consumer个数比partition个数多呢?则有的consumer会是空闲的。 + +![img](../images/design2.png) + +而如果partition个数比consumer个数多呢?则可能存在有的consumer消费的partition个数会比其他的consumer多的情况。 + +![img](../images/design3.png) + +那么合理的分配策略只有是partition个数与consumer个数成倍数关系。 + +以上都是基于partition的MQ所带来的负载均衡问题。因为这种静态的绑定的关系,还会导致Consumer扩容缩容麻烦。也就是使用Kafka或者RocketMQ这种基于partition的消息队列时,如果遇到处理速度跟不上时,光简单的增加Consumer并不能马上提高处理能力,需要对应的增加partition个数,而特别在Kafka里partition是一个比较重的资源,增加太多parition还需要考虑整个集群的处理能力;当高峰期过了之后,如果想缩容Consumer也比较麻烦,因为partition只能增加,不能减少。 + +跟扩容相关的另外一个问题是,已经堆积的消息是不能快速消费的。比如开始的时候我们分配了2个partition,由2个Consumer来消费,但是突然发送方大量发送消息(这个在日常运维中经常遇到),导致消息快速的堆积,这个时候我们如何能快速扩容消费这些消息呢?其实增加partition和Consumer都是没有用的,增加的Consumer爱莫能助,因为堆积的那3个partition只能由2个Consumer来消费,这个时候你只能纵向扩展,而不能横向扩展,而我们都知道纵向扩展很多时候是不现实的,或者执行比较重的再均衡操作。 + +![img](../images/design4.png) + +基于这些考虑我们并没有直接采用Kafka等基于partition存储模型的消息队列,我们的设计考虑是消费和存储模型是完全解耦的关系,Consumer需要很容易的扩容缩容,从现在来看这个选择也是正确的。现在去哪儿网的系统架构基本上呈现为基于消息驱动的架构,在我们内部系统之间的交互大部分都是以消息这种异步的方式来进行。比如我们酒店的订单变更消息就有接近70个不同的消费组订阅(可以将消费组理解为不同的应用),整个交易流程都是靠消息来驱动,那么从上面对基于partition模型的描述来看,要在70个不同应用之间协调partition和Consumer的均衡几乎是不可能的。 + + +[上一页](install.md) +[回目录](../../readme.md) +[下一页](arch.md) \ No newline at end of file diff --git a/docs/cn/ha.md b/docs/cn/ha.md new file mode 100644 index 00000000..e86ec0ad --- /dev/null +++ b/docs/cn/ha.md @@ -0,0 +1,21 @@ +[上一页](code.md) +[回目录](../../readme.md) +[下一页](monitor.md) + +# 高可用(HA) + +QMQ分别从两个角度提供高可用能力:分片和复制 + +首先因为QMQ不是基于partition的,所以很容易通过添加更多的机器就能提高一个subject的可用性,消息按照一定的负载均衡策略分布在不同的机器上,某台机器离线后producer将不再将消息发送给该Server。 + +除此之外,QMQ通过主从复制来提高单机可用性。QMQ将服务端集群划分为多个group,每个group包含一个master和一个slave。消息的发送和消费全部指向master,slave只为保证可用性。 + +当消息发送给master后,slave会从master同步消息,只有消息同步到slave后master才会返回成功的响应给producer,这就保证了master和slave上都有一致的消息。当master和slave之间的延迟增大时,会标记该group为readonly状态,这个时候将不再接收消息,只提供消息消费服务。下图为消息发送和主从同步示意图: + +![ha](../images/ha.png) + +目前当master离线后,不提供自动切换功能,需要人工启动master。当slave离线后,该group不再提供接收消息服务,只提供消息消费服务。当master出现故障,导致消息丢失时,可以将其切换为slave,原来的slave切换为master,slave将从master同步数据,同步完成后提供服务。 + +[上一页](code.md) +[回目录](../../readme.md) +[下一页](monitor.md) diff --git a/docs/cn/install.md b/docs/cn/install.md new file mode 100644 index 00000000..13cd7cb8 --- /dev/null +++ b/docs/cn/install.md @@ -0,0 +1,159 @@ +[上一页](quickstart.md) +[回目录](../../readme.md) +[下一页](design.md) + +# 安装 + +## MetaServer + +负责集群管理和集群发现 + +### 最低配置 +JDK 1.8 + +-Xmx1G -Xms1G + +为了可用性请至少部署两台,并配置一个url用于client和server找到meta server + +### 配置文件 +*datasource.properties* +``` +# 可选,MySQL驱动类 +jdbc.driverClassName=com.mysql.jdbc.Driver +# 必填,MySQL数据库连接地址 +jdbc.url=jdbc:mysql://
:/? +# 必填,MySQL数据库用户名 +jdbc.username= +# 必填,MySQL数据库密码 +jdbc.password= +# 可选,连接池最大连接数 +pool.size.max=10 +``` + +*metaserver.properties* +``` +# 可选,metaserver服务端口 +meta.server.port=20880 +# 可选,内部数据缓存刷新间隔 +refresh.period.seconds=5 +# 可选,动态生效,broker心跳超时时间 +heartbeat.timeout.ms=30000 +# 可选,动态生效,每个主题至少分配多少broker group +min.group.num=2 +``` + +*valid-api-tokens.properties* +``` +# http请求的白名单token列表,用于控制权限 +# 每行一个token += += +``` + +*client_log_switch.properties* +``` +# 是否输出所有主题的详细请求信息 +default=false + +# 可以控制单个主题是否输出详细请求信息 +=true +=false +``` + +## Server + +实时消息Server + +### 最低配置 +JDK 1.8 + +-Xmx2G -Xms2G + +Server需要将消息写入磁盘,磁盘性能和机器空闲内存是影响其性能的关键因素 + +### 配置文件 +*broker.properties* +``` +# 必填,metaserver地址 +meta.server.endpoint=http:///meta/address +# 可选,broker服务端口 +broker.port=20881 +# 可选,同步数据端口 +sync.port=20882 +# 可选,动态生效,从机同步请求超时时间 +slave.sync.timeout=3000 +# 必填,数据存放目录 +store.root=/data +# 可选,动态生效,主是否等待从写入完成再返回写入结果 +wait.slave.wrote=true +# 可选,动态生效,重试消息延迟派发时间 +message.retry.delay.seconds=5 +# 可选,动态生效,messagelog过期时间 +messagelog.retention.hours=72 +# 可选,动态生效,consumerlog过期时间 +consumerlog.retention.hours=72 +# 可选,动态生效,pulllog过期时间 +pulllog.retention.hours +# 可选,动态生效,数据文件过期检查周期 +log.retention.check.interval.seconds +# 可选,动态生效,是否删除过期数据 +log.expired.delete.enable=true +# 可选,动态生效,checkpoint文件保留数量 +checkpoint.retain.count=5 +# 可选,动态生效,action checkpoint强制写入周期,单位为日志数量 +action.checkpoint.interval=100000 +# 可选,动态生效,message checkpoint强制写入周期,单位为日志数量 +message.checkpoint.interval=100000 +# 可选,动态生效,重试消息写入QPS限制 +put_need_retry_message.limiter=50 +# 可选,动态生效,从机一次最多拉取多少数据 +sync.batch.size=100000 +# 可选,动态生效,从机同步数据超时时间 +message.sync.timeout.ms=10 +``` + +## Delay Server + +延迟/定时消息Server + +### 最低配置 +JDK 1.8 + +-Xmx2G -Xms2G + +Delay Server需要将消息写入磁盘,磁盘性能和机器空闲内存是影响其性能的关键因素 + +### 配置文件 +*delay.properties* +``` +# 必填,metaserver地址 +meta.server.endpoint=http:///meta/address +# 可选,broker服务端口 +broker.port=20881 +# 可选,同步数据端口 +sync.port=20882 +# 可选,动态生效,从机同步请求超时时间 +slave.sync.timeout=3000 +# 必填,数据存放目录 +store.root=/data +# 可选,动态生效,主是否等待从写入完成再返回写入结果 +wait.slave.wrote=true +# 可选,动态生效,从机一次最多拉取多少数据 +sync.batch.size=100000 +# 可选,动态生效,从机同步dispatch数据超时时间 +dispatch.sync.timeout.ms=10 +# 可选,动态生效,从机同步message数据超时时间 +message.sync.timeout.ms=10 +# 可选,动态生效,是否删除过期数据 +log.expired.delete.enable=true +# 可选,动态生效,数据文件过期检查周期 +log.retention.check.interval.seconds=60 +# 可选,动态生效,dispatch文件过期时间 +dispatch.log.keep.hour=72 +# 可选,动态生效,messagelog过期时间 +messagelog.retention.hours=72 +``` + +[上一页](quickstart.md) +[回目录](../../readme.md) +[下一页](design.md) \ No newline at end of file diff --git a/docs/cn/monitor.md b/docs/cn/monitor.md new file mode 100644 index 00000000..4a81e280 --- /dev/null +++ b/docs/cn/monitor.md @@ -0,0 +1,55 @@ +[上一页](ha.md) +[回目录](../../readme.md) +[下一页](trace.md) + + +# 监控 + +运维可靠的消息队列服务,需要对各个环境进行充分的监控,QMQ在各个关键节点都有埋点监控。QMQ提供插件式的监控接入,默认提供Prometheus客户端的接入,如果使用其他监控工具,也很容易定制。 + +## 定制监控插件 + +## Server监控指标 + +下面列出一些重要的监控,还有其他一些未列出监控可以从qunar.tc.qmq.monitor.Qmon.java 来查看 + +| 指标名称 | 指标描述 |tag| +|---------|---------|----| +|receivedMessagesEx|接收消息qps| subject| +|receivedMessagesCount| 接收消息量|subject| +|produceTime|消息发送耗时|subject| +|putMessageTime|append消息耗时|subject| +|pulledMessagesEx|拉取消息qps|subject, group| +|pulledMessagesCount|拉取消息量|subject, group| +|pullQueueTime|拉取消息时排队时间|subject, group| +|pullRequestEx|拉取请求qps|subject, group| +|pullRequestCount|拉取请求量|subject, group| +|ackRequestEx|ack请求qps|subject,group| +|ackRequestCount|ack请求量|subject,group| +|consumerAckCount|消费者发回ack的量|subject, group| +|pullProcessTime|处理拉取耗时|subject, group| +|messageSequenceLag|消息堆积数量|subject, group| +|slaveMessageLogLag|消息主从同步延迟|| + +## Delay Server监控 + +| 指标名称 | 指标描述 |tag| +|---------|---------|----| +|receivedMessagesEx|接收消息qps| subject| +|receivedMessagesCount| 接收消息量|subject| +|produceTime|消息发送耗时|subject| +|delayTime|消息实际投递时间与期望投递之间的差值|subject, group(server组)| +|scheduleDispatch|投递的消息量| - | +|appendMessageLogCostTime|消息持久化时间| subject | +|loadSegmentTimer|预加载索引耗时| - | + +## Meta Server监控 + +| 指标名称 | 指标描述 |tag| +|---------|---------|----| +|brokerRegisterCount|server与meta之间通信次数|groupName(server集群名称), requestType(通信类型)| +|clientRegisterCount|客户端与meta之间通信次数|clientTypeCode(producer or consumer), subject| + +[上一页](ha.md) +[回目录](../../readme.md) +[下一页](trace.md) \ No newline at end of file diff --git a/docs/cn/opensource.md b/docs/cn/opensource.md new file mode 100644 index 00000000..79cee4f5 --- /dev/null +++ b/docs/cn/opensource.md @@ -0,0 +1,10 @@ +[上一页](tag.md) +[回目录](../../readme.md) +[下一页](support.md) + +# 开源协议 +QMQ采用Apache License, Version 2.0开源协议,[参见](http://www.apache.org/licenses/LICENSE-2.0.html) + +[上一页](tag.md) +[回目录](../../readme.md) +[下一页](support.md) \ No newline at end of file diff --git a/docs/cn/producer.md b/docs/cn/producer.md new file mode 100644 index 00000000..e0c0da01 --- /dev/null +++ b/docs/cn/producer.md @@ -0,0 +1,107 @@ +[上一页](trace.md) +[回目录](../../readme.md) +[下一页](consumer.md) + +# 发送消息(producer) + +## 与Spring集成 + +### 使用xml配置方式 + +```xml + + + + + + + + +``` + +```java +import qunar.tc.qmq.MessageProducer; + +@Service +public class OrderService { + + @Resource + private MessageProducer producer; + + public void placeOrder(Order order){ + //bussiness work + + Message message = producer.generateMessage("order.changed"); + message.setProperty("orderNo", order.getOrderNo()); + producer.sendMessage(message); + } +} +``` + +### 使用注解方式 + +```java +public class Configuration{ + + @Bean + public MessageProducer producer(){ + MessageProducerProvider producer = new MessageProducerProvider(); + producer.setAppCode("your app"); + producer.setMetaServer("http://meta server/meta/address"); + return producer; + } +} +``` + +## 直接使用API发送消息 +```java +MessageProducerProvider producer = new MessageProducerProvider(); +producer.setAppCode("your app"); +producer.setMetaServer("http://meta server/meta/address"); +producer.init(); + +//每次发消息之前请使用generateMessage生成一个Message对象,然后填充数据 +Message message = producer.generateMessage("your subject"); +//QMQ提供的Message是key/value的形式 +message.setProperty("key", "value"); + +//发送消息 +producer.sendMessage(message); +``` + +```java +//sendMessage本身是纯异步的,方法调用完毕并不表示消息就发送出去,可以使用下面的方式判断消息发送状态 +producer.sendMessage(message, new MessageSendStateListener() { + @Override + public void onSuccess(Message message) { + //send success + } + + @Override + public void onFailed(Message message) { + //send failed + } +}); +``` + +另外,MessageProducerProvider提供了几个设置,可以用来调整异步发送的一些参数,默认情况下这些参数可以不设置。 + +```java +//发送线程数,默认是3 +producer.setSendThreads(2); + +//默认每次发送时最大批量大小,默认30 +producer.setSendBatch(30); + +//异步发送队列大小 +producer.setMaxQueueSize(10000); + +//如果消息发送失败,重试次数,默认10 +producer.setSendTryCount(10); +``` + +[上一页](trace.md) +[回目录](../../readme.md) +[下一页](consumer.md) diff --git a/docs/cn/quickstart.md b/docs/cn/quickstart.md new file mode 100644 index 00000000..0421c474 --- /dev/null +++ b/docs/cn/quickstart.md @@ -0,0 +1,30 @@ +[回目录](../../readme.md) +[下一页](install.md) + +# 快速入门 + +## 发送消息 + +```java +MessageProducerProvider producer = new MessageProducerProvider(); +producer.init(); + +Message message = producer.generateMessage("your subject"); +message.setProperty("key", "value"); +//发送延迟消息 +//message.setDelayTime(15, TimeUnit.MINUTES); +producer.sendMessage(message); +``` + +## 消费消息 + +```java +@QmqConsumer(subject = "your subject", consumerGroup = "group") +public void onMessage(Message message){ + //process your message + String value = message.getStringProperty("key"); +} +``` + +[回目录](../../readme.md) +[下一页](install.md) \ No newline at end of file diff --git a/docs/cn/support.md b/docs/cn/support.md new file mode 100644 index 00000000..85536ff8 --- /dev/null +++ b/docs/cn/support.md @@ -0,0 +1,10 @@ +[上一页](opensource.md) +[回目录](../../readme.md) + +# 技术支持 + +### QQ群 +![QQ](../images/support1.png) + +[上一页](opensource.md) +[回目录](../../readme.md) \ No newline at end of file diff --git a/docs/cn/tag.md b/docs/cn/tag.md new file mode 100644 index 00000000..82c64bc8 --- /dev/null +++ b/docs/cn/tag.md @@ -0,0 +1,30 @@ +# 服务端tag过滤 + +有的时候一个topic的消息里又分为不同的类别,而不同的消费方可能对不同的类别感兴趣,当然这可以通过在consumer端进行过滤实现,但同时也增加了consumer的复杂性。通过tag过滤在server端就可以过滤掉不感兴趣的类别了。 + +举例来说,有一个订单变更(order.changed)的消息,然后订单变更有很多不同的类别:订单创建(create), 支付(pay), 取消(cancel)等等。那么有一个应用只关注新创建订单,对其他类别不关心,而一个出票应用只会关心支付和取消的订单,对其他类别不感兴趣。这就可以使用tag过滤功能了。 + +```java +//发送方在发送消息时需要标记类别的tag +Message message = producer.generateMessage("order.changed"); +message.setProperty("orderId", "12345"); +//这是一个支付消息 +message.addTag("pay"); +producer.sendMessage(message); +``` + +消费消息时指定感兴趣的tag + +```java +//不同的消费方关注不同的类别,现在这是一个出票应用,关注支付和取消 +@QmqConsumer(topic = "order.changed", group="ticket", tags = {"pay", "cancel"}, tagType = TagType.OR) +public void onMessage(Message message){ + //只会收到pay或者cancel的消息 +} +``` + +消费消息时跟tag相关的有两项:TagType和tags,下面对tag的匹配规则进行描述: +* 如果consumer未指定任何tag或者tagType = TagType.NO_TAG则producer发送的任何消息都会订阅,这是默认情况 +* 如果consumer指定了tag,但是该消息未携带任何tag,则不匹配 +* 如果consumer指定了tag,并且tagType = TagType.AND,则consumer指定的tag是该消息的子集才会匹配 +* 如果consumer指定了tag, 并且tagType = TagType.OR,则consumer指定的tag与该消息的tag有交集就会匹配 \ No newline at end of file diff --git a/docs/cn/trace.md b/docs/cn/trace.md new file mode 100644 index 00000000..4a9ae36a --- /dev/null +++ b/docs/cn/trace.md @@ -0,0 +1,27 @@ +[上一页](monitor.md) +[回目录](../../readme.md) +[下一页](producer.md) + +# Trace + +消息队列服务的参与方众多,生产者发出消息后,可能存在很多的consumer订阅。如果一项消息驱动的业务出现问题,那么在定位问题时能有全链路跟踪埋点,将会启动事半功倍的效果,QMQ通过接入[OpenTracing](https://opentracing.io/)规范,提供了完善的trace机制。 + +## 接入自己的trace系统 + +## QMQ中的trace埋点 + +### Qmq.Produce.Send +生产者发消息sendMessage方法调用时会添加该埋点,该埋点携带了subject, messageId, appCode等信息,可以用于定位sendMessage方法是否调用的问题 + +### Qmq.QueueSender.Send +消息真正发送给server时候会添加该埋点,该埋点可以看出消息是否发送成功 + +### Qmq.Consume.Process +消费者处理消息逻辑会添加该埋点,该埋点可以表示消费者处理消息吞吐量及其耗时 + +### Qmq.Consume.Ack +消费者消费完成消息后发送ack回server时候会添加该埋点 + +[上一页](monitor.md) +[回目录](../../readme.md) +[下一页](producer.md) \ No newline at end of file diff --git a/docs/images/arch1.png b/docs/images/arch1.png new file mode 100644 index 0000000000000000000000000000000000000000..e283af3b34092a9e4d144441351b86d0c12936ae GIT binary patch literal 27728 zcmYIwWmKEZ^EU2IaEAwXin~K`cc-|!OK_LsP$am!6?ZM}?(SZs<)8k}d(QhIerL10 zbM44AGl^DFl0iixM1p{TK>aE!sRjW7r2~Hch5!Y=zgMo60Y5;xsmX{#)J_whfnOjx z%j&v8K;TmUyFq?c`|lP4LKNbwq?m>`01(dX;^I*~zhBGzvrLLWIW&{u`t$_?Awb(KUJhd6iCTd9p3wWqEyAA$F6; zx6ttR<8Mb#Le^##A28?O(^rXmV>pMRMG$Uo>lK0`in zE0@z|UJB2c`^3O)nsZ^5^g=IS%O_ysB88@#?Gx>R24k7F?^hbuDK+AM7|-+8Xlohc zDaJwfXJy((HsP-)`A%b!e;x;$W`w=wN-`9)K72(y_}{a^^F?6+!VL5u&6|74B$9Gj zKAu`+7+Q`~^aOrXM7D*MtrRQH&7%zPp4K5 z42^Og)4*XgZJzhX-6&b$?>)Zb%!BTmfI5{y+Xt(Te`DtS8*{2C!+|UdW+aZpZ71Nx z(B~*EQRo@}BZt?;%;2(lK`Gezn}yZ&zsv3Z17FT1p=l?gWDDQ@SlPdL@5xa2E=ljl0`vDrl+j3ivrnKe-nv}y{I3AqpjrXX) zH&vJ4`_YZY# z^5nq!4Vl}iXNlyeW&NAaKfRwXyP@bKTe_B6M+an|yzy6aqohG!w&JvtF57zlTuBo- z!1$4`DF@!CfH98?*B3F#+Dn|OmPzMR_QQ3e_wN&Hd9ZvJ7%UJcZx?l}-Kow1$ct%2 z+8?6H-;dL@jrbmZed&)jlehDGoRit(Bo5d7{5>i{zE2TJ45in1LiYC-KI(|@cf(S^ zr8!y3UHd@-beL^fx|p1G+sn3}3^27-;HK-r@Wwae-Rg%+MA!O_-cAW*A8o#WqaLRd zc**n*gMD@#$IJRo%$?{T8-edvUSQaF-t_1PY3>c&hx~_U=&s5PKYrM>s3y-`-ve5QkkK}zWYSz=545R)+sZ$IfBe3KxM2-?ovaJ`V<$%;b&J>g z8x{bG!n+#u_aQ(3vfsa%O@J{4p)*p*y!bjP1zTo}4KuZG%=DSVvf%l{d*=NPTZR(}>^N+g# z$?G0^knEg+L(38bv#juKAGD3@l5Em)ISfBFb*9>>1pJZATOTy~>58G(mXtZ%Ja|(u zaR}JpDQ)plTCK}VV-!-rip#wetV#vk3~cRi#Bb8|9YGpEIz40y*zH*pVKA~-n`gz- zCJUW_@hvMx`Wy_H{4NLaJX%q8jCEZcneIRUET1h1`K{7l4ogbKtQ{E+>WA>#T;c$; z{eOCCjR=STvHS@pAk)M9P=Q@c%|k0&1y2U&@~c8!+7^j@@x=>RnPs2+g(I?eUR0Ht zGq&j#MYHUE-?|ArOqRhYQ^A0dHo9_`NuPaK)-jKthtudWva4x96Zvq?jroQPqXjK& z3p2PW_cJph8-WDa@vU7w?CO4!wN8LvYTZ}kXZi=%Kbdp>llgko;wGsH#AiXln$MQU ziUp?sah8?vL)J`i3CP~O#3PuePiqsDlC>FIS1z@JF7$|@!S*D7S=B3wFh5l1f?Xg7 z@!`E7dugXC>mnBWHbg57+1yN^v9RQC-QdoOWR^<9-mOjq5IFhaHh)ejkf&Y(5UZD5 zk^D8FQ4jMfYSSzxmrZ=?irM;WO#hEDGchN-H}VdFT6Zex4j4|7ZI-zd|8~ zDPq|##9GJ~k>&>deYG`PeeZ)8?xUw+i+&TQs&>5(=YfTypYQ+~zRESY!RyMMvELj( zYyqABp_hBZo~ooy(82%$I#bdH+VF>&Nz?niQj^Nx6KIC!*ZM&;^O4Yf+FW-m-fx75 zn1g7e3J4vkgQVlCZYJhnO!KkLyzzd1Jjg)F;XnMRsA0Ys`1$9h$?h7A>EsuYCGHF3 z8rVZ&1<TuM;Zr(OAbq)K^NA&`;?%gTCVF z81P;-f$dX$B9u1?;&p2KkgK(hbJ=_!dg1KAyry^x;qt9n#b2hOJc6P+yC3$k@TGem z7C?I7vaCrpUkMkN1v6C!#B>Vxh>w~x6bc4m{~?@wM2RS^qVHZrjw3eJl@nI(#sO3L*vLz)|n`Iordj?938PScuQ+RI`u%tQjfRtu3BgkDso5#I3}q* ztg5K^c>M~FwGAyp56$p$sM8SX>=Gux=799rVFWL;q)b0lN~)~Ku%dM(-IKWWTe^kV zqO5Upo2*tU2HuQvw)=r%TQz%Niw~iq-g$yt3$W$;_%hUhTChJSR^mh%Vx7Dp9EH2_ zvcdj*zo|@p^`EOZTapK`T3O*3>(lk|MvFH*!)U8A)pe3H`40=sV4 zhQklZc7sAXcHo|pZiyVb%Rbd;p7Z^vaL0w3{9mHmb30~Z{AJz=MH|nQq6)!k_oR7j z*s%ZDhTrvV4`U4s%>PrD%^EqTVJp&0!!#cDkC;)Lw${J z)&8rwG(9?0N(;6CH&Y+iG3jQHDF>mX^GH2)?V{#Xdo{ zfbFk{?&1GTkXI&5`JVXaW0}Yv)^OZ`couZkp}0VscJ4|ekyDrIF1#kn7Xsf^l@9yC z4K!_tJai|08Z{Q~J+Byn@PucTB`i98_V+z6hSMgCOriSTR*qTjoK@ZUwQ2}UKVgDO z_|jhj^gx#H;KgW$4%5%==jKS$qv~i*cVJQE4zQ+fP~0|U;IlMyX|617Ur!HpE`>RU zi~WQQQ$GPEFy~<>y1bOl&Jg=-seP#$vy$vH=LR; z2+*9DYYVVHBZe`PhF90_fgNOG>EaKf;qxp;-BtsE>pl%Gg*?oUIAoj#T2Ap|K}{2| z^V0HU(aCOI4pe1C51b*Lny(Ki&y zIm1x2qs73z)|xYoYF zpO+;UPxm?nT|udeZKA`A8t>e;<;nOe#HtQuNDoBG$G2Q1ZZdg0rePt$PMCGlS#nPQ zPDIhG5<|}P&36yaR;208Qa{#x>Fa=bCAQ z)Mo&>nGGIHImB@NDlCgQxhf^Ds7Zt%kWvwW4hdxUdr)QlyI3994752>?yUgq20pSo zY=)AYP`5l(7Wo3MR1V_Tk=3m9@l>iP-dA#-$_^i6eIKzEY|!)EIZ`AU%tU*M_9Owl6W<9=_!PZ?|r(Q zY&mtYx4>277f0|rRXhg;eH3~1`YmvcB-V98VoE#z^*Naz!yKCfguI>21on&-#=5*$ zSGdSQ&rJ53^PO89#Vuot&Rvpi(*K5x(Gg>VfbFMP1V*vUMo>jVRBXn90!z|y0?V{i zv&&jK;Obi#K9r#AiGI)8qRbC46sMY!R)+hCP|v=CaaU~{r}B*$`O5rt`=#flDgTeX zrcb48*e=qQVl$UMd9RAh#ZrG~cACAGI0#?|Y*$a{*5)7eHV;`QwXId(l^I)y^y9BB z;ui1V{C@NK66Dl+R=PULa40nYA4KL6&MJfy<9`Ajs>29mKtupn1SxKTwc>)#E-V zvlWERUdy|mMv3s>5Y57N``)6=HFiCH6Q$U&z=SZol<6uB$FoU^Y|RN)9W8enw$qTZjp9VA>Tsa7*cCUcxDFm3AiO zRbevA{Xs+e`}d_+MBF3HXT~#w+u#WlITt$Y6)H*;0QLqw8Ef0kE(Buo!$v{$YQqL9 z#HgGP6e^tUAo>XI<$XF)ha$p=9r{l~2C6cmP5Rsx=LQh#8zH*SMM(x%Da<+xkh@)} zxqF6-D|kMls@P#n>c|Yx&$Xl>ZezyiZg5KW`CDe1>N#o;Z(^lDm<51b>A;p&+Vc7L z-CWBsb$JPKWl=1!wYkLl7nqM=Ikpdzqi|VsAl}9d2N3bO><>W<&jQ%bpJPFO1mp#DS-rJuc^CL^dhaC;l85F@&SfK%i;}a} zw2;b}1hy#U=`;rRr)xGJRhrXTJyq06gr+5X=3acytLBPUV?1u@uw2UGlKDxEa%*t! zQ^aA&T*l@c+D1M{yW(Z=+$iZC`$BN?)(joVj*@GIaC|!^VYXVsNEe@&`M*pE&P`Iy zGI-MmlI^6LAJn`QH6e%q>=>>}tVj6#t=k?Y3Tl|QY`H?Y()sfvD*N-heRJP|QO)ct zdF5IOw9sJ38qYyNgN6 zFTnnS9rcdNH>h1_P%y?lBwQdUeS1(taCOGg$Dq9M*Sp5o1)<(qrMyZyiBV!Vls-7s zQtd(s%fJ(X9UHiy*ip|`OoxrTFwtQwpcVFe6_7|Fow?ge3Kj~L?D9@5A4;lB*_!AQ zGOU^1wh;Z&r7X=sTFDw8R<$aaGHs4zQdrN!xlEMZ-%O^*-=<==sWDY_>7SV~(!jL6 zVx1`05{P-Fem3VsZ)Bpp#(y}^RB5+##saCBeO9lf9kBJH9g0sD+e9q6q;hB%2)%5k z43@#*LQXx{_A9IxZ0d=rAQ|jQhKhxATAA7AI}CJV|Gqond1LnSh{ShvSFU#Tz_O8qk_F$|G!;W*FAiO%qhVE#U9vw~7^darDwK=}lHtvm~ z=)%7ljbG(R#LHP@bvLji`{3d?xw>aXS6j-=@h~1O1;PGgeOy@oh2sUgL zE)9@|7qy*^B&OdMLqd>`S__D$lTc()9UbLk?2s{}XH+JLsZb-Qw7;U`y4G>owH5C? zX^T@dMe7Bd4Vll*1XIcp7+3zGlT(Iu4@;gp$eQFg3CF8Jk}MuTBAnBxWRT{;P#qkg zXk~H=R77-d{I?6-{J7K86aC?49K%cp;smf4t9M~<181SQvUrD&6ce+L%!dVjH5?8w z1Neq4$=vzoyf6$^HLZH~N9%GA8?vMvTKrdu-Y5t#BmuA@m_N}0$cm~uwl&jsY=CVG4nhBlwZ%06$<|7@EKb5i1DNbKx?x9p-K$|L)`sshKN2 z*{>W-sUnjJJxuAEPmlRgzz8bB(Xoq?c)7~zsDO|6BXWE?23bi!3^ASrg~wMC^W54 zd%A=mra~r#1rqY&>>Sa`hFsJZ6eu6G*Cw5m=uN9+*ho+c1;bD47C4}&s|%BK7%hhz zQeJK!r>-e#+@J7BAULeq!IrjufUhie@CZSl~lnqL37{0jo|3&&mf zj?BAu2J*Gv;l6DbbjVZG5X3dMln0m$*QiwM=9Xzn&;$51!D4B68D+-?n?vE;xKe~9y+KVR%!Zbn;rtPuC{a7( z_>gs*T5B9uC~L_RZu1)u8gY@mTBI!4az_9$=KJEjz#P=J^B`(^L;LJo!6Zsa*siz) zh!b|D!%ZLc%F$~+0d@cbB_3ngVTE+MW6_t9NWq4L5o>V5RL6#C2w$5}rxW(m@=u@i zVD;W?*&;_!h4l!mVr3cT2LxtlmA&F&PusW}7OpLND5ZPx&v^vcFzc_uh0oSTZB=JZ zXc9&qmEg^hg*#Phm>{B-lJ-kTK6GcJbL<2M=5yZBd)>g6#_sVDuXw3|7Gcewy^z zi5&{hCF^j~oT_J6-J%RwEf!Oa!0R;~(0K8GKv>1Bg8|6f^{YB}FW+YRT{nzVLqrM} z=axgJ-=!ABv%BLA;!NcKVHX3{c~r<4oW%CZqj$a9x*->?g}33U?PHzl^HC6r_V&w4 z*!-JVMQ9Wx2#eo8u6<4~2@-7lpF#rWSYV=%NGD6gV%QoJta_HO?a&)jvrn-D%l43n zHf_Ra@BaX2YgF~^9D+)3)c%rS12*dpu6`r(vlxHF{1}N0maE`!P znx7HNCk9`1?s5a`^n=+B$6_pJUsKmqp1jDmtwlx_9c_+XecZnU<3lCcf2#VIy9jXH#O@P6IN3gZyjMWk}lHx^J} zEhGaasOz2*PmBO!442>}ioE@`O@e@mIlgRM6uK+5VGF@q9IoHk?s$i&g^J;hJ4f(O zXMH2Cmk=jpy6yHeI5)?t4#hKjjq>Koj9G#Nq802?3CpGl1ym6MXM^hvDH;w?z9AQY zRQ8-r)3rdh;g_90=fLR5vnWDiSja_{J!$GqjGntOn$&7F|P{HEH3XV zNtx+b#2|6;lP9UBWFufwaN@MWE>Dr-Z(fRn50i5QEr|Dd<}Z~IEm~Nq2wXG)mge2? z`dUA*0D!^aq(b)^t|zIT+ybBCPLDAn>;2z;utZ|mPhm7(&J#3ZTK<%BN&7rw)t&AJ zMipHiDpwKFWLIg!1ZHXT9S3qiI|4b-EF4Edx8W&ZDKkU>HU?uoMOx@-z&2t<(T}dK zItO9tt#ZsDfD-J11)+1YJhpB)4u&H&w)Vx<)LsbeY*rHB$)yxvszJVn?=NR7l6l-e ztS^b;hm>F$y3a`wm!47{j3&xpnf#Q;5jGE)|4REcPQ3i)@<@raALvkoIY4n;>drwn zBdBtScHF}ozzD}XRDh5~XEl|uIhw1cCjV3Yz{1!Sx|lky1!HcsB^~8b9!#tJhoR&L z*TwxIPVa1VW{V;&A;iN(pdq?&1jZwpBz@`=$%uf)tD$c%d@$R8P8EO+&zx_e^d9F< z)FJc1DVa1Ql+X1Bq!p#y+_Rq<_Cvnt=c{Km$o}>LXc?>Z}Wr;g6Ti6Zj&;l38S_ zzyC63FrTqkQ~x8u2m^cTY>7x05?d<-VXX$B<}If8o%6@YoZ1$Sn3hvsf@5;T_*wYd z<%@=d1TOo#_a$y7cNLyVseLTW8zBPoEoP>*f|HZI$5nj?KMIk5r}>K9_11GXJ0F@< zLjRVqLF6+Q#)v*cJE}SongcCKrARkz7dItjd{SaVj)PB?K(Si7D@VT$loVO|u)uXY zw4E6wWQ>0>r6^R)V-phTPpTWGW=W#*=smN+0-{kf{32)m5}H;~QxYHJ#xP6S0%f4e zSW}ALaH}6XSpD){5$efAIsPy&Al9L~2V308b^Xa!Tk(Y_tJ*G$)nW}b%TX{X&z}a>fY9%htVYjJMx0hWc^vckYz^5u>z9VQV zGdD!reaFtLkzgz8bEn^}Y!&m zY@=R&Tl@Z^B+W$%NN9%HJi-QWMp6$a*LeVO#mLWr_p2`HlB!9aC5T4?k8}|r#=1o- ze1~{CkQNshkq3-@{9=6=a))yjN1v7#GByWCL%{|#GdHa3j#%HN;$H0FZp~hZ06T0z zXp)sY2umb8=Q595>2Chg?!~%5NO9$#S5pd__Q%_Uxt{CMtP&)xzDK(nHW2OtaKj{C zn|I*sHfcD;s}56AUGvk=Yyo^g_uuP zk*j^6U}8(Am_38}z|c2X^czCz7vzGWO%f%Uy)}%VxD>P$WXx>UKt~hWxQTU{J~Lih z^;{e;E>*kws=WltQh1XxK7fMry1AjxK&I)7e9?ci@1oWUXb5 zc&&i4RQDnnhUP5T+}dAQ=&w4*VSiL%Cg=4t-)6fvtvuuD5Ex|{vNNJ;EpsU#heDzj z<{lg%u;{qc%~WBVAMRRRuc9lLu0wZQ%B8NXp*6V{CD$vt5B&_ev$vjFMuAD&5eQP| zArejBrT9JA(o4AFv6VFWZc)l?6Y)`I2<@s>75NmoKcp}<@ZQ}D(tmb}jl7B*ILVYx2m7`qWcP`FT4L(uiulHXL6yOi_0))&870mSE z(R7>dki$`%FK|+Tu$T1!BO6Fr(K$vq01?qobmO}X;X)}z@}$mNfh8>cIVekqdO{Yn zXXcg^ff}}Il6UnP{z+%0{(H9=ps6S$@2~k9(Msa)x2#+hy6L`TaE|n9KZ$!F639&w zA7dC|di(Yw$2HqrP*L5*Ek^!>yA{n$aO7GKbPt z&o=ByPepJFBNJ#q36*1Vh=hsIqG8wEMDbG0EIDUdKoAEQuT45P$tw4|uafy!63{M%FCKDk(nK*s(WX{8U{_Nm?t5WF zMyeJ};DwZ?iHgZJSW8$+a^qKCxxQP(4j2_CenjH9Y|i&pq$+NsV0(3&7q<@yA7tY0 zul)MBt#43HHCgj*k1VZLSV^;le0DVa)AfZo;waT%+#q^yLESlmz# z>7BehM{-1mr^tWtnk(^EGQmR{7w6k7l^`aYzS;{jQrl9}ynt?{_fvl6`d($vV8W0|GIC~*X%4B>7FM|Hog3Y=|C~#|DRwcCnZMu474s(&lRoP?^gCg(2xSWP`eDsn)#QXYb!X zEN#O|&_!4CYBq)l&Cct?mmL79^PE0g&WI**2jw)jgLo1o{{ zYBu{!Y_U!n8srhPojDo;EC$Y^$RIg#V%~PnwAI!e&76x!*dJG;=ia{X=|8+8(t_c? ze~2Y(3vUYZ-K%JJKZzz&vkI>k&@^PYh>mg|yT5@$NMo0>khh}eSwKzv;x?^)f4#)= zC5$0r7^&{_GlITlY-?%hpeepBj+Qrju&6=#C`%WrzK z3{_}cB3GY71JNfa3=1OwGluX)xgbiWq#VTg8lH_Lt{4_u>r=%|oUUN;sNqc|;m z5Y33G`z4F&h;^;-e@RIiGA~O6d4Bh0fN&9^?s_RNYrR}>~2C@L#obM+Y5t!(2w{J;D+ z>tx{xk733zM3F@UK`|fvJhD}rpxH#Q*0FJ}FPU-jH40WZ#-n3Lx_kJKB6+piE{(84 z1%$s_?N+`}KCW$PFGG`6qou6q7}XRCKoz^fPFOVgn(n?|j{1j4MKrO|um2F1Mp`Kf z#WYaSvQqBas&lf7DMb-52>^-~nH!ty4*z|>fg*pzK(NOHj$*EH=*D%eftFPaVwrNt z8HZ}M2kb+UTRyGKGhC?%qDiN7M0Nh`M*P_bok$y59Qo}q6^UEb6Q-~xh=@fJ7l|$-Qag z-Mw!=c*EJgU~r-RfZ|SvL3l>w*bXF6HVL(owHH8#`XbI{$eJxaG}d&YI|+kli>i51?hSJQL!3xn7P>3GaMG^D&dm@2wTD1WKQa3Z+(xz&z|Jb6 zpshpkCIM;aCvEj*g;?8wRq3#h!~rd)n- zM=OR6aqz~;g$k2K`@JWA1}1yY-t>ipp-iBYHEr3h^O8IY-u(VAfQU{ zHm!0@u}PX3lOX+&!ljBCZaf+1;DfF|4Bz8tPSbJ&D65b)?g#rPYEn@gk=vYTQJZo5mltRv!}#%;E>=o5Gm z>UXm1!+FUmKKAD>Pt3O%7BBG>2#Hz;@8z@6ngtw{#gC_EernhwAy&`&~f&Go;uUezApHy(4b%GWAGc{||_&0pDD#~V0#`wf(> zPJ?UT@L)q#56-JfnDD04Ic$|8`a$#^a4EOwUX`^7BfGV*nMBsZOm@=xJ*|WY(P^Lh zFweRcIwQV*7YLVMG`Ncz?OKzYo;(9KrkbVp19WVlOx59Yc_fJy0)}z4cg1z3lR^U0uy$9%NqrpZM3AkwIm+2z+|5*>KX@$d_5-g=m?1P>~2MrnH7$Z~M43yZF^l%e%v0vOq zZpND>vUj-jBGgh~BetcqGdnHa>ST3{4Nf8k>*d-8b-G@aIP02&Z$blQBUNY!eCqwG zjK ztl7nzy)FXJdPr(l^(PtkAwj{fSqY6|elpp(de{`(Zv&Fzvxi z&B(nS>73gBWMPJ9liT5hYrOsfOXe^zxUd3COO;p%F;lGop-|igT6OFp!k)3nnT
GoP5ST42k_AWf`KGe9h}Gr^k@CydK`er z9WC=QF4uG{Q{LuN40>3oVM4+$cWNKvh|PZabARo9piDbQwl=_ouA5k#L#M;W#-P&e z)ps|mWqBMh+Md79AS07;q?956wygAG#{SF#Ub|G78IrOZH4c7ZVv^eeqjY@J?gcCc zI3M7> z<=nX>rjja{%P5{ojN^W%KnyPAUGXxr-UljARJf z4fu`sUR}xF>N%JzSITc|CTUi#>XBRDqE7`(s`jb&IfBT94}X@9l(0-6cP|**NBeS< z(fe@(&XAA3-}Lr+RAC_7Aq9fKbW;%o;yw|B`Sy)7SZ=KW!Paoc96igTbKiedkgDrN zR`gtv*vLbBNB!`>AJ6~$ST@KwKmP+U{RJ8!j9?kJAjNoP6jr`ID?b*5d)FlSE3c>ItMwEo>%npB2koVs2d>2=66t>e&vbA=#v7 z#5ee6Pz)s&v&s(-_ulTmo1+L$o!K#pBX{L+hO%S}7r8`yPh5VfP z{77UjI#2gjbv63eqd@^#8j}dFQ~yWMZq$?97l~~+_f12iMf!@D2SiCuB*Eab24#Vd zrEY>8$xkJbKki?QIyJIDdPYY9Q)t7Xd(q0$Yo6S=kX@Vwc%+&y^n3uBbCTUUWkl|L z8INVt|71echI{iG}D z>_Yp$wvuG|;M|asQH~vjp&}w(TWRy)QzVI%u})(s6Xd+eCI7Upw0=!1mOK27(aV{jpZCk`*|n$u54d3s+JkEPWE0(5j9^*rtK z(eW^{d8$ju*Dao=JFh8ca7B{7Ke*P#vkx3*pY;H-2%`o!)*=kTE^XK8Zb4KvN z>tUKlWbT*n)o0VY-^LX-sozI*gZap!mR^xMQR3^d-Qicdct?};6?H7O4dI8&CUxWd z^-H6gmgRMM7Cr~6@36Q!w_pax(bI#9K40PZhg$-sQs)y_pTOeMvrVsfJ-)WQ^NHO|Ajqcb^ZSRWL=-3#5243Xm=nES-(6RqO8;hvVuabv+YEldLPX zmb-A!utkuXzMm~sEhSIj9+f|B%D+e3UB3K!C_#+)Nh;)fZ&xS$y0}nP(=#IY@Jryv z|DoIBZ<`817J6Op-_k6rEQ`#`mQNJ4U$4&Zd46P-LNB*`r9F2YEt6U~PFmbiUl2}T zPiVI85<8a-%%3bDoy)n;A1WE^I0B#Uibj9c+l!6$3R_HYYV`!)wQf8V!ut#q1;q>E89zmNRS4yzijNvjoxoRhmu4xi0DziWC7=LEjvA3n#r zBh#GNBeRVrZ~FxkX6Ij*z2kDXW8klYc9WwM9bWSkC)YWlF%k%c2%+n@`R!(eUVd1d zWq-5;_dP=y`>--2U-y3Yt5O}rOd?O+FsF**^A(E2BwLU_;JXDN=cxE;s>^gcWRji_4$pLf#&s}@vWl74zBHeSv zJPx-=3+ZTsRuk8?T#x42W+)K}4U%!(R5-XNiBrw7xj#grVaa>xI@jT!%&dLOO9#2y zh&J%x0|3jz<5BdJ0ULmSy<53mbCJ^1+#A)_w;4Rt>TWQm8R7=*IpGAhRA$?LfT~lW+Q*i5J?t1vdw?vOf30AkUU%73m3t;A9AXL)G{VQ-oYe(2>gu7$S*sF-!spRA} zxzPr0t)T&z@jNNP3?LX^8OT(hm*mh3NI-FiHJ{`kGzk>(t;M`io21}vLv|s~gS)wvuHjQiU$BybW<@(?P4c@_}Uy02}8uSfHN+UxT zgHquhvvB*oYBeo>-e``riFzk(HQ2!0j*D9C%QVr|S7l}G=A@j?si!N>Et{I7x0}I- z!Ue+R*DDu(U;!+yv4HVL9bY%4K+qU}S2!w!_+M`$%g!qw$|qb%oIu|^C2S0I*fNg$ zOINmV!!I-nnI>jfu}$-GjA`HVTlHJcDi3-#t;P@wCFCIg4*L*$zN&{nNt8|t#}#Vn zn1;`<4FzlrcN_%5n6C{iDPiaUh{g)1k$Fzh$a3tBB>poIF;>V9T_#l1gGRpFU!7jn z=vmgYOB3!xt)`5*zxAWOlw2C z%Zn^P!s(uK)6#_Kmv_#GW;oAefO0sKfo<81ey6C0gQaI`*je`1f!7R2UlYK^UNlS| zi*ojXieO-#!k1ogz|rgD;yBR^=*iA>#Y={%tFGWba=5jF(#Q2<)A|>EfYIm<90ceK zLpOda$-HJAZqLBNSDkN%obvGt5;8iN{on?=_~n;rdH*S{75`HSvX}xENTX8~lQWOT zNrB&-aQsDd(IhTdB<47~d{;k=s0CyqJ~Mf=RL!D=|K25T-1Ywf_YGnULc7-lyaKzU^w*S z7D#-D#Hqz(6!v9C5IfjMSApLs2Hf*iv^&9c{6P+nlC@~$eX zu^WPc7b(wM1Z~DQdKX2NfY3j%_^&(`nQL zINPd1y#6?#CYIq~1yPwO@g1ek1YEl{T+HM(wjAH?Ij<=8hyV+So8xYaZ& zdPPov9XHmH$embA>T|=;Z)gNOE4SU80jvKF#~uIS;@l?O9epk}?YwHui5}TJ+xTR zR9T&Wsmty&CEA5dN<~UKaN&p-NZeP?e@g|4Ts1HB+{HUpx31f|$APQS*Ri1lvJUr_ zA>oO4Z5qJys04Ca=+* zt-&sQ6pe4ozedH|vtekb>C!g2Cv5u#E9gtTeEF~FD|bt>Dg@p6s;OKdJew_hca~V? z7dn~k&9t9XvyW5<^+EaG9}SEw-M@Q+wk&ME*zYT_rNXe`ldgfANjFghG_r-e zoLHia28o;zr0@}{KSmNrH$-icx;)CDAh%-}YflGaE!P`IUgn3ByNqxXTC`GvXD{sU zzj-;3;5#o}8ZE%rk9p@5gIjLLK2FQdZmmqiXnxNfm< zD*UnEFFLJ%eK^MJ3cg8(?t#L9oQAG{%hJ!70}{}*o9BtjBPpr-dM-r8dO7%bc^5bI zi;2C(_O;rF144`GjG3vs2DR9MMcbd&X^>4_H8`+1lN7}J*?<$-)@7ct00Str@F@LS#8bX>ITTx`vZ6P|o zURhGRWm#0&HKloRA2U@~(8KvaBXqluoI$4*K|nB`{rfL~Iptd-dFd}vUF_1s$c$p_ zI77;>Z9Z6RO?%r4zV^zeFvz*deV|u2a3?hyV9|1PXi!^obqMVi(2db=W?RzTK3K)B zGTS1|-xZ|yot_X7u%rjJW1ztcQJJ!#QfTLa7ldQ75OU>o^hjA2Q(!qC=$fmp- z`zMm2!<=E!0*_*9s8<~UUoH9kt8@9PBrLCx49j9ucQFb29hj4&`w#%K3M8T!iz?(G z4kc3aW-C?(CSQcz9Vw)$NLDBlHdXvMU2U4~POKK_&O#&^y-aYyZ0*nA4>_u##y&)3 z#hJwLRI72lvg`29iq53}C~~ytiY?E~l%DMjMh^w){;#LA4vVV$-acIe0wdim-6c67 zB^@eV5|Ro^HzOb&QUanwgLFwFARrCW-BQvG67Qbpd%eH^xM1S2&pvyvb+3DU_P&qk zbXLh{DNZbiaqJD(+8d48E{2q{&m_i1`71ASevIo?P?;|rsZ`@Qlzt&2(yM8XjRKpS zQQGq(#$fvlnmZN{}pl~i!gR{qec2gs$25+GzZY%4{7uz8dwg`8xONi zD4l-p&N`N`&%2W3c^ovX=wm*0ZaJHDrMG(GA1g}2lV)b%9H*I$QX^8IA9^! zBS49v2H3;i)K0u`lOv)a;LUWU8pY)0QO89p5LcL}C&}f0p^U$B?I|C$H7htAP4ahl z8e3JU{_2m~MVE2adm#k%9b7<7`Rte-T3F4b=K4Fnn~=M+q~j+JKjBRg;lT(@KCWP% z+QybGinM?5(<4XH@FEJu;dL@?>&WiNlbFb>$x1ze`cVdcX_P0W(=NL($A5cu7yLr9 zLNP098V5-#Uk!ZI1_rMtYP1 z!85CmO!;VSwRDgH^{tqWOdrOU7?Zfd=ygY&xW^k5n(Nmz52}tdeN5wk4uBtR!&XMh z-d)x4cKL(s%?h%$n}tw@%KagATk&cjXfdhnRMYI6tY}XDmf-$J_=&TD*<<}XBNfD{ z0Q>6=OoD;3Hw+azd?yH|GoN&~%QwE#rEG&4yd~{v@K4vxG8^2*T8XLd>uu0xS6ks~ zMi1eUS;@bHe^6JPZF*2@qKSo3yp8!stQAQMjn^or+sj&?{0T48<-@Njnkaa^@<|ri z9-XdFxmh8;+q&|mT_bFmMAouef)(`453OOYw)4;1Fw>dGBy0{QV%@0E+S)&?e@^bn zb0P4R?UkL;H=G9|czCCusV#157?+FZI<2Wq{?vyHN5nT3%1sLfpsO3sW9>PUdEJW71FN;ndSKtksQZ*7cS)A{$o>K6-_qy zp`f#$81?7LyuK~PTT}-ee|C=b+ay@IYFn}O%PRajhf^%?19-;jm$ry7dJGj%jrbvt zw$Un>+`;8HZTW$eugcm?;)vAGWTsGjja|shfXKLcT;pRwq^j{~u2Ey2!Dt1Bz>jv) z*Q)MMEKaAQog=L7a*HWeU)~9q%ge}c?4QgD&+GQdAMsz2SA7&9q1eJH90}QQj#ECe zlhb}zE6;+UeaXE39MV<99l+f{bRH7^xP zh>>mZMvEX_UJSp=@A=UTi8Vc+M`fkP{;Q2OR*StI5ry33Y1c^$r|!$QUMK(S&6G(9 z^8Qs&Qf(%zHeHnwdT+LU8!W`#mP3@3?@GJ4XCvmjTU=Y8>@#D8dLM%Ccq{cp8XBxj zOCt_*kD?vA!XJ>y^e}Bc4KNJs%_WM83XV%@hE=w8!LSLlv~--dnV*F*j4G!v3H-qu zAv0Hgdt_^&ypZSOltnU?DJ_)$ACKjA0t!0mzZZs%l!lKbk`|oW^*LosD}vdl;i8_l zqy!^4_Jeuxi)pt5FY^9-J&5b%NEUw4#>L4K6ZdC+9bOSb=b1=X#w6A=x zBs6sYk9TAD9iz~n>HL$?Pd&zOR!6JC>jg}lWix|4#~F;lazP`lwu6O5)cp#&o>tY> zgA2UDeut67*4Uh7bH438eL;Q1TJZ+Ip#e&0Z3f?*7)|GQYob1nMiEo)41zCu6YUc) z8bt}CgZuUlIpc=NJKGo84qV_h%2%N3)?qN-!l05@BGy_f9L#Y?nWt&O4@dp zLT+ELptXToJCUvmv~qe3Y-t+Cix$NhIu$&%09x3^{JVQKK8-mUG1-kHeKX$)<0Zyy z_CJbIYy>{lm$jMXIm#`&h{pbM7e6`4jNbz8vQodxxZj7r#A2xVGjn-VEb4LcFHFti zIH{NX=B&d85KaE*8wBk96>`#@J#1VQyuX3)UQTM7kqI~V8(R3X2H~2xCzB?;$x(LS z#~YB-i7yh-*?ZAaT-rP;51F83SzP?>W5>&5H9qDyYM!J;Ri5r_zE-n$mhKyS2TuLm)$7?&yz{-o zy2ZaXV~)pevr&amvKS1%zNjndzIGCQUH6Fk;BHI#-s)%6bU&h7P|#Zy@r2c)l!{u& z-Qpijo3$h1m&BpJl^I6F@*}|-&Sr`D@mhvg?-#k%#!|}3h#JBQm5zl_EJY}1zhLRu z^Sv{Ee6yu2D?c=Cz*t$a_x>ZnmL+bRY}2Dzhf$^Wp%l~*k0&*GF9|&xMCP$zl}fPBggdP_o`u0cM1!WtQ&*tB zTeYCrV)snXndR|_I35*k#8@F`)#phy#5;oHWE)h-cIZ%1UFKtMZg;z~&jNuO0D0u_ zF^E=Zgx^CGmn3Ows3*@?e)5@7-J@>d+3AuJ@^j;T1No+sqAPc8TsJPFooFlZtx#+r z?1LVdb3obo)#yG#hirmHw(M6VR-ljL9iZ|P6=cNZ4h&>Kx7cyrSVfFf_)q1}SA~{EJn96cl)Banm7XRlou{%jlo7X9sLSykpRY0qyW*u38skr2w%lk0; zW}y?$v5%BUu#TufgWyYm+*8(iy#u+#$GP!?;x zb;-o@4&9Y?cU8m#T$o(>C;;bNh4T5eKVDv8tlIfj97$i8Q#0@px&=JCa3zXkf+fl+ zd%oRuY~Hy^QYgFjVU+pJDxL4Hz=&2Amrp{BI_@)$V7lKpIdztN+&3yh`PqFR7!va^ zR9Ut8v-27)?BUItxTl{meDH8$G%T_%JEDuWDj$h7jZxu-lizHJ{pgkKhF4iA80!Hm zH`Fc_!{-AP4~xO&(6Wl+i`)Zs#x#IhdX|Dhf={p0Xd#%LY?J)s{38ai?nPJP`dV&< zHXNw)Nte&0Zaih6_}zj0FmHF8UXI|{kFYmrjKsb%U|kr7Gi&%-DocCJd;(kn#4;cq zvFb8W{???Yy?WE3S5*A8AB%uvR>IJ?wXE77LcxTk_d zP}OM&Bv!bxVjr7T==>W-ha?s=9_ASDbaBq-+i%>#bZ#jee_agyEce@AuzkIm9fsl@ zP(<4)sh*x##rq$)#_(g+KjU+>I5fnlHdmVc3ITQRHICqqUS1E6Hw$`n@tC-<;hH z$h~@ST8+`cYj7E!Z(8w{K5W<5+%805;8cE-oGc)>h_UETR*=WiNjFZ%37>V07G#-DolAQG?@qL46V-_RS;`K)*@~O!-?x7=Gf8bQ+|vXl zicGP&w?qZxW^a30?fGafVKskZ*@pr2r-0}Ag>l|4Ja!L^`Vl+=etV8$9f+L_3$zbm zgm;*2-EjrHu8CZUZC^g92-ZJRBYWUywA zf}vZIG5iWd=N;$|#2JRcx2e}04?Wb9as?h>nd_|_y?k|hn-+fq{UTHTi`ea-*bGf3 za%i`O#9`v{d|=`#?;HR_>zDHmJVMpP03*AsP|-!NH2_Qcw_82@T9?$28#mXKpZzP$as`P95NX_cM;^h&{^7_@t==6i~;Nits7%(IJ>40!sS6v^) z1@69?h4f>Nha3oL+zV$dw{k)80{$8=!QrCB>&)rVc1zNzGb@dtjZHj!8V&L*no_TG zwxGp@NZ?QD1Mq-Z%wkG98B{$sWdQZoc5M5h$D1UM^hcq(xh*a(0)G}_6JXPr+wfg@ zLx#sMd^OQrN#sYZi<8pW&q6*V>D-M_LJ^QzNhJoSY0O08y=cqc7j7|Nkhln*uSoth zqwuX@RW^ERc4aKdqCJh7#X!ZZQ$hPa-rb`Pyyyd^EX_73^59}_hkhD8Wu{d!yL#uy zs*M24M=7b_xi_3Bd?xE2a~ z%v48kdvYguP<78868=H3X1XKEZ2Z!apsGJNCJ`2l{Of1!>fF{8@Yk!u9{~3TC}ey; zK3!TC<)yq0@JT4FAV1E0DmyCA?fqL27uFu0q(b79Hc*}YB^CJ=B6HwP0}L?p)xvA= zFd`{(c`$YnZv0r`Rmhl`0dM=wqA)HAJD58fiLPYxap0fkrgz`08`gtXp}0b^ry(86UyEkYD(_a ztbbG+0+2c+neJ-%{>p3`a+R>EG6Q)Cxo6I}sXKy#>ugr9?o4V61h@5){;B%CXI~P> z#eyfj$0=Z#rT$eUGqeUan{;o`hp2M;l@IKK7f|r-Pd9b#P!*+`cC^7&T{Iz8U}44b zMv;BBCRdM1ZW2E=0MLnAe4smo=atH{W7fS;){dV3d=&=mLm&aux?zjh?t;p&{r4GL zYDo_#y>G+vdT9R0IY@~$)8K_%DCU;#JRS%-g_JiR_Zhy*e2$7^!yChQDst9AiIHxu z#1;%ErX|6Ar1=?#0gSJ^a#(50`D{N=-F%Nj|2#*qOmf>NnY|05g@B!+f&q>dw!Y)K z>z!iRyUVpFx&9{}(~yg$!22k^)=!R2TTxbiHBpT`YUjCLSE8}_cQ|%q?8S)Hnp`l z0!|1`Zhdny_)Ppq;Euh;jbJ!!7mT46i#<@k8vg0e=l9BjQ9*qvZ3QFc$1nqGwB0U3 z)M@-?OiNaS9LZ0*dYr=>0Qh(1K*8oJtr^Ad`jd)eMdIO(Oa0c@vE^f_>?>=c4mG<> z{CcCn7!m5OR<$H3-~Pt`ukkK?Vi%!AYvS5PDmL%(M7PRK@@!o14ba;$Is$xct9}q5 z-EmJk4a{)3=K<64F@ntZ-Da!MAHlu@Ayvd-ETypS%Wj~cmsWOD*5jXyni&+Z|6?$=B&SIs<_S4!Ck#030rLS&(7nJADpCy` zswndlCO}~;Gy}6?qW{}QQXWkaE$kz6sUiF}AE-ikN_ME_ z$z$tC(qmf%axs0OK%k;M4GFwiqaTBGk+YGB!b(idFq)P@J5@`T$JqYykj;r6+5!u> zvt%(ZEcv1@!Ln=2a-mt8KZ02d_WSWN>n2?^Q$Fy5Ndn8F7VXlA(9(zgA10n{(CTq9 z&sw(hZscM#BFBS=wxdmcm%T;13+$E20k58sv3vEhrYem4Wm~+$SNop*AMkW%g)sJ6 zsbY!$wPcMC{;WPd$<4m!4+!Q@WnrBP|0f9|1sID-VDe^E+08r;2C>((^j0CBL_c3H zPQ`XWei(ZU-*eU~n0SqT1oX$H$5E5mqA^_GAGsa>;%QnN`Ls>JcwQ3f6YKcPoDZ50xxxv$(DmvT57zWY9VP zTNdDPZi2QCz@GeX7hsY5E)qAfIViSN zLL*^f*q7jk*?p?hKc7JI(c73KcpS%=WAC^T4RST4;K#U>HV)~x8NXdZ$Fygx%<#xw zZev827EwPYiY?xq*ZnGYENfs(Y=6A@`J_n8KQautJ}2OvE;(& zir?&f^NX)$3>7}ZpV1);H+MX-sI25;q`U5A6i+_c)RlcDhAy*HWm|tSBsR(H5&%(S zs6)qI^y{r(yiN(iO$W({_pe-D@o|+e;NF8Sza~7Aqyf#<*e;JHks9OI4SzmM`JGMZ zE8{u^$M+IHgY{&B9fX|N2!W79Y~3CN%9e?Y{xbjI?xHsfB6J9e+gGzfmkUxA5qxX+ z`?$ax=FW7$*hTIL0KX9%=?!q3UqJQKA;eiT5Ey@iScCkKBvfmiP^>3zI6h#iH~hrM z^+Cs_ZhqJ5EL6WHNQ1c>W~e zV}Dq2Yl@AzP4aU(|I^>PLy1OMZ}2k9RkVuH?IGM1D7W~Xjl&av;1FKe@I2V~aP|m9 zr!BzxjXt4o>Q34c^G%ulZ1y!E=@RLuBayAV{O40Y?vlf>anN^Q3^#M2CTnw%y0Noo z8yC_v+Mm~mA`f-R+_4Z~=V=A3;mS7ba^4D;AGhc4LqV!!qO{;KHQp`Ro#fw!ik-`A z;C+gqrBAC8mgB)98vQv5=#d}jeC$>`Bg#OKjaQFDvZhaoJ!p1A#rkV1SSI^eciSru z%}Rys=xoZIu*`ra%i^+R5yUm(T2mx4+qaFtCivbVWXI=kDS0~A;Yp%=64_p`pO4&B zQs%@%*dq8ns2}6*CSfT_s6|z&xGFqg)`Td}20M^ud@-)G7P-eNLFh!A9QtA_6m zctLn-v^jR=_|jSLN3*l*wcQIN=0KM%kVcd$b9~T@BDRy-aPOaromn<0w8^j50l0Cf zj4EGeCfZ{Ly!B}Y2nyEQXt<(>K`RJZ^q8`8u=IE**jpSKRM)s24Fa6gKnQKjwzI+| zA00x^!L9DQa(KKmPGU(eZ1YPdI|Bam!8ibEMnF)h6F$-@C59;xQ|T;6mIp=VIp@TP zQi;7baizCA;>fF1^k~k5+F3Dhj`!6M%Z8qJe zNrX2_)W5>lhE=AW)}fy?@RU@=F(^AX;^kWTuF{X5p~HwEf>IqJcKGsVqu$f-j^~>8 zi_Nd|kk`qvxAK?sj|#{Ro0}#0FirV)!D{$IR@?>0_A{VLrjQ`S8BB6#^+Q=mudCYSV)ekzy> zvg*;=+P#3>%i;nz5F9!2|rTO$hsEj%nSK!YyLl= zD0$(a+d5x#Q9zsDPa%=G)SajyF^HWAP97X6AJL)QO}qGo|7q|SRs)pXpHY@5s>shU5yYQx z;VvV)F+y1Q{HlS?8Y-$HiatbE3_L=Ko#Ijxh)q z|CctZ#eFGi{jB~Nikp-Tgac)!UZku_1TM*=2li|(8Yn1Iroo}a3iH&hyEf5wvzW#H zE<&cNi1)}ys{Or?!kB6$Q6tUU9>$;DvvbjFSs&;taaV~HML`40&0hJ@sn9Tp*>Clh zK6&JNz4BAy`@Y2lK;xu7MSSg(aXn>FabkFD+7Bk29ga0vQ-slRQJo3 z5BeJ_UjqY16;n+_|8iGH(~a#oPa^63R})jB(=KZQR^6uW76{6<{QVDkYtPFx63o5N zk}sU=&Pu8^=dg>@cJ9H-7gAf-P1M^aqP^MG03RN6#0+4#edB>Dl#tp!G zmDdd!fpGi)Kd)vTB7NF#F7q-!t$Wb=@9p+4%$nR;)nuuE^Y8S{PcEj+GEF#idf#M0 zBGW<2c{|2y!1SqE9f-DIscP~f*$zd{3B%<`17Dvk+Q>}uwZacCea)qFQ?43UBm4Pk z*P+ha{DomN&P6^Ws`NfytF==L{0^rG*_5SR$gHw3g%0~CkM_5ye=d?>smLv*3AWe9)=Qf?1>}Fp9n^czK2J{T_fvW9KG#K@K8BvEThbVbaOPh zwJWrjQMWs}s3O>Kb7LylT;1(fk<9uW`r9qQotL%j2+06Vs6zmZ4%`aEAkGB#|Lk#3 zx9Lg9(`OyXkQkV|y$d_>0O3D$6a1V7b^GV<7`~dcV6D*PFI;`g5$phj5rdc{3F*P< zN~o1NzcWm7*GKdOw#K zkxBM%t=st-WafTHRR|?KN3S|g!)}Y0&)sg%rj`IdpEG;OAzFS{FLDD3=q_(LH*(kl zM#bt`hhjywn{T|-I3*Pwcx7*zkCX_$*ypFEfO`SXd|05BQ2^P)2hOs_oo_(;sy>b_ z$%2@JEi_G_skkm}R4td+Ei)>q(u17!>Hczh!mn)oSDPcViSfMkH={I}sV zVAt`Nf!p~GHMkEh@#wQl2O5MM#chRNL25Y&-C7oi)5yaQde~#i0kUuaX)Dxaz|cgB zIN(GG)ZsFqxhT{7y8#I!=@Pghnm{@NG!wa6ZFMA0M9#d&6G(_Wz@;qDl9dyKwhj;d zod)FcWHfx7$KPF&CMaNq&h;x{z;8{DP_#F! zZ~*GtvMLG@_<%l zv!&32peH5^AOU_N$$fksd6;W6 zI(JzM*Nq@8BTA=dO+NlWUi(<{)8pW!4>G5W_?+RhAP+!F+{o0+o|ioxh(geMf1tvN zH#xi?2C|rzw7atTI%Yk!Q-z!%b+$lKKQhx#gOaEs#ar%fcF^OP3l$!XYHh>ms0(1%$NI)lK9r>IJ9u?-ULWEctaE=Do$d;x91_1M=Jf;7;6 zOXI^)ld?J>O4(swQ7}-lG13&!Ii$%Ke(Z}l1xJxYcK9QCOwQ>K#q53G4G3Vfe!Ro$ z+OBGgZung;JZ=~CcG*r?-7`!A;gDY`wBS430z^Z3!EDC48?a9rgMIRRZ$wKn4lNQv z1R<^u3k>S?ZzRz(K~;<5u2QkIO+;Dezxz#M9|ICXG3Xf(K_84`NNFd^ zoI45K=RkiNVWVX}1E-$EXAxI*G!*KP8cYJzKjD25Yv{jXYlac5xBPQ-XrHY_%+R)Q z#G@wr`ge*FBv$p@D(Zva!!j2GgaMcDGhR0b+Y7Z=;q|VAXn>G4e%y}!!S#=(f%}KF zku4YjuLXK47qeJbDC6|Pq&=q(JGZU4y%8vndT(~%qP>-vmD}$PlRxtCFRNCf5fnLs zXqIgDAx%oer-l91D83X3SHI}f#GkK5AtLY53M9TBjs#m)@Eq2%hsZzm$N0%lk@HvP zFGC=`7Y%3*@H8v?CZ%J;-ET~S_MY!%?%wqN;_qART6s3_1mwsiY3cfNh0E-UOU;4B z?8+LHH7l8D40eMk={_y1!?M8>z?1e zo8M?I`Rn@al%wKuiGyh4oE6bvqB9&0Joja?K7njp;;eD*e3?ip@s)VWZqaFUuOOz~ zbv+^bQI^>(NjGIzU*h#g(lJE(Peo|ht#{bGqCOVI)hGpsIp{OPLhDb}?8O^ks$RXK zMNcs$zoC*SuRpS~F3h-d5QR=Cn`OdRuk##S!e2jEGZgv)qM`SM(mo?{UyZ%8Li+;x zkV+uz$5TZ05W#`ic3sJjhv1~5UED{3yE#n{eB`Ail04bhE$pq@-gAXwVWln8U=n95 zIQoY^MOAy-%az(Jt}ekEpo`{^UE$_ff>e|WWsXx(KMST@FBfJW6k@eC6rvGv5~-)8 ztg+RKXjjzprA>B7d81!p4N`B@gHt=!Vr9LD!NPddMaHAr14d32A&Z=QfwBMaTf&rN zT#NDF_Q|8iCBmf6N63ijQ{}f;9t}`)I(vIHUx1vnuYJKee!jLcWkFv-s z;Zvf$8ApgUkSS%UHV_uNGjHStK_t@3S X#cFmJQ}^IhFqEf?nhIs|7V!TEIcvX{ literal 0 HcmV?d00001 diff --git a/docs/images/arch3.png b/docs/images/arch3.png new file mode 100644 index 0000000000000000000000000000000000000000..55c507984362aa068db6aa23997fff6b6ba4ef8e GIT binary patch literal 26945 zcmX_|V|ZlI)~%CtP_b>JW81cE+qP}nwr$&1C+Qd+cGT(QR-bde`y+X(D*LHjd)3@) z%{9h55~V0F0S|)%0|Ej9FC{6e3<3hC34FW`00aIGCbs$a2T&Jf31N`>dAtkY6KF?C zEf)|F4Dx?Jpi;_YcOW2wAX1`2s-B=Xc~HKqV((uq7eTDT(4iRKw?$IONycJmJ^N87 zybJE$?E@OW=bgt2_Aja;`aDXh`=liVOT$sT9iXXOTfcl*7=5>F-OqZq$a)U7{KB^0 z=2=*NEWGqQW}RhOf9d?+D~=>YqExp*L_`1p zSnwc3Tp-dR#NXGGQwUI@K?`sgSlEqPEfqz@0~dG_EZ{N7kdq;a_P|306Ny0}K3@jD z_`DuW6MyS;>%f2#4=X21h6CRU90aUFp@9BoIs!$VJ7xa#?5xK%E4R3qBty2tfZBZR zvers1Uwltp?DW@OPhE!ejPkc5-(7aPEYM&<&%h^CudNnJTH0Ef40;V+CkTxS`8*tK>@NGg?h=Uv3No^fhl~MP zY3b>inLke}3NJT1P-f59TPewjN5D6gh{wZ5MHNx2QLmexp2lIdM9ONjUej*3)smM- zfM;{q_ucJuvh=juZtZ$I&9MS7f1c(zwA=4VtL^o8417GySgpQiK5f|TwEO+J>2Pfp zkHc}zBv6j6?teXs!RK*TME|`q1^Nw=jFM8vCef9M$Z=l;J`_QU5cn+|2#bT1U4DJu zeihc+)OVE!03Qx~@ryzbVG$$Zae|=h`QDGQn9pXiTA^f-?fcyJ&FAqL`2Ai>#I|Cl zl+6U8i^F0%T9#o5%hengvHJ7*=hlb~;(9y|7dTQ9%LM_ST&gf!dhT0hg#NFGIS&2M zPReT^Z;!+>FuqSq5;+_W;hT_%1iU`a*Ih1W)i|{Qhr(FD2)|w>LLlK2?KbN7Y#RC? zJ(N%LJobxz7l}lWe6A=6{Jk>xnx3C;pO)qpL_&&V6eI|B6#*7eF2HuP5w!RnSP6O! z`h4d|-XY_P;2E@9U_*lJzGKUP395&XZts_0ar%;n8U}u8IPSl8p?|R2ZL_koW26g< zi4^eoe{{B2DJ#=mD{-|-!y`oetn1hp2EQz;&SJ5sHy9x3Ln{9JSnw5F9!V>GykQUk zU)IH2QCV3@tx_&3Dk`{aSyrdhq2J?9$Ln|7_oBRD@b#`yiHxke=JCUCqwZ$4O!!Uv z4OqMC{HM#s!17Fu0ukM8G!g+MplG5+AQB?n)M++>#ar!mHld8%MWVPLjYgfHTUv$> z9B)*qL?km}P%JW;jPrhdJSti>==UWCzZG)Zq+Htg`ufhK(;4y8C)w|`*EQ#jkBtfC zMj+q|}a?U`glNDLk}@jh#4LcF=Fwt7KYxpR;Tka1VtDc zac622hA2J485K!CPEt%lX4=Nt9F6S8gi`6m*kqlJqX|Yk7Yryy(C(4QHeMC~{-hFf zr}lesJGAs0D5w@GTxbQ6A8=j>Vga~bZzA)-A*lRdqHGqAOhXBSb19oP9YSQGko`8N zi5*|V2%pcK!)_-nBNOsP)$Vz392uRgND5)74wp}_e4J9kA|8RQy;h@r9m zrV3RXK)nT=1R!8B1XGSjj#4NU#8K&x}rlpZsr9 z84QaRY%!m!?|qQ=RX8A+bl9lZD-n-VCMf#)^W%##beQASs2L3EtckxujXV8YnsmhB zQwSpeph38dPP^^US4wi{NaL@~HXDuf_XC>Pfjph zWyR;|d}y2a&FAG;8Hl#tpXZyU*?2szU{Wf1Ry288SyQbf98GUiy+-r7>_}i`Wrs=4 zR+kbheJ;bqEu*qK-?d4lP*9aK5?cKh!qn~e<)T|g>S0q9F|xNVF^4G$QAX^Ip_z(f zwO&2Hs3;hRPDd!H6a0in0}lm!JCTwU_&4d3211T+6GW#PQ>P)eRjP64L8A}Fcpt_! zSPRLgiArM3fgr7(P#f04I5|@lK3qUUS1-fgtNEQ%(pD~G=^Ywo zDs$Z~j=C)HL3upi>}0?=6$Di*NvCalw}ZWL5FsIvz(r0F<^3zF%Lou0O8E*$7QER# zy@0>Jt-p)??pZM$ASE6)3;Bn#t2&1jy!cCIw|e(DhIqv`$H5 zDc9EPdF;7uqnsBJq#teiAb1QH+6oNvo)mPm(9PJ=XB^iMDGC85pQB*F z)T`0K!5~1N50PbJ*Qj~ruAigiP73^RrmbcrBZ1T4fGz>P;~c`)`w!#QM2=wY=0RP;wpITP58uG{2lYw$a6C7w~gP1abVc!FzF3)Fq!+&*Q_nbI>P1SwQp zv+MaYE>u@(2eB8%ctFgrl6|paCG#YSlne-H;CsUUvMR`{yp(^6d0h zv}h*`I~~)Cg-?og&Rp+kxw1RsO?5_6QV>^+-LN*XAti=SC^l-7LDefxBx#IVwCw4F z==~zc2!WA138@UlT$RjI5Wdo?2tNj+o+kNbb(x&wbyWE`S7;CDKELQS((U1bV1z^| zqdoYH>SLQU(m|E*0?)->tk>=HmuZm+&ln`58o?M0IwYf^1P-XW)sV<;aHB_xTHUVX zSL_$&3Hy^p6heKJ z%&3>ph%(~_&&r618JS^Ru>&T38B=&l?p{dQWTXj&G!;j=&J_`k<76Vk_`po@xu)vo zj3un54;By^di(RMH? zCY^R@xCGy%aN^~p%Dnu~Ml8JbsZ#Vv3@tEJRNedM9S6z+Y0d%Fx(^|%b6<{K$8T?V zzSB;TDn@=zB)klzC3zp4V1mkTyJR(I zt_pg}euynyWQAruQb7?TyQM@H%69$gzM>Zyi}ynLWRpF$@4leG+Au8$pQ4k*78?8x zR*dW&R(T1_2vW2UkH?vK5b*?Ae?+Eic4_J@pD2>bPH_#68IGCO4B^0(t?0bnYL#mc zt#`#S@CHAEeHa)O0QhPuhy}B@3`xOqJ)kt^J!+|dBszCrEEEDPlY|h_G3!vq@qAmjRB*goR(>znEL)$|7(Osn=bpUg!K)>jw=8%}a1IB;J64to5+UKFW_ z|3^wMGuCxie;l?p6^nQ2S#yE34u`at7`hW`ES^Z=+=%nGti0UVX&5hh=<=)}V7xiH zDb*NIOsz~=D)xPmASBC8qbc4T8FyA&3(5YoMXB`tH%Npo2HK%(Xj8WK%1kH=Y8H!Xzf>bc4BMgw~#qzwEk*;IaC@^>t z+lI(ybxGn}8}Nj9c-DpcS}o>6p)>|R(D#0dWZa#SIz-P4dPao76){a@n02B1*eD<3p$~ zFkK}Azo(_!{yzvA*x}I5hM1NKYn;3WjlsiSoRf{Ck;UU(Ho0ukerd_M_R?S;;S2*G zCkop4#Usi&H3?;>p-yUsB)&Ip$jYD!qRwER3K zBlLI}+MG1|X| zBJgKz7uR))bYa>&9&sOXv(ZqeLAgyCwd!a=A%4urMH=#Zz~;m^lPf_?W1rlXNkk(C2FT706S zaKhK0lK|J7^`8&lCD4v}5L55og{liH@76Rv*ShWg*oen7r#id4oy9K*d>)MFIN0nL zm(|rTAKm}-`trL;^ZyI16xK~V5}YuIfsc=$I`&^?E9G;uxF@VSH_0po)tNXi1IWk?~=l6S}xJ zNKm}>M^uq5-}qrGjX75bGklROdlS3#ncCsJTf)U7MDJN3As#?RPdcQ0W zhr_C>s??pN{y+q+yoGITabY5!PlS$W8)2cT7C)k+pA+52;Bg6*-l=eRiNFPUnJ`}c5n zn9#)J+-?HF&im!kqz@U@E8BeS;GzP7Y;*!1$1}-jMUu(j4`Qk`^MH@d$5qw3TJ z8ykPyDfdvUy<0jaD@#j(|Nh+V_vKb_q^_R~?Qly34Tl%k}yzO!~6la)jEymaBi6)TH5(k$_%>BeZO?a7B1 zy{LAD9Q<&c*@KG=ZP8M8)*F~33*2Cm`AY}Fs$O6$Vr%spY%4Tjjm|QDRLipW$u;lk@WVBL_-05Kh^Q; zt%qCoV|Qe2K(Sn(&X;yN943B3b_?66YL~agB3fA{ly5;r$w%pB=wWNG{4QcgzK;b_ z6p84DOAqAO_xu)Da0qhMs(r4ep~i&9Y2)PvEkQ|p$jgc1sbLgqv0iqydPEJ) z=oBnAD#L&uJhUq&+85M_v({WKCxyaE{3wl78Pri9>YUuT*0U;0ST%jJu1mvkRb!o6 zL+_TZwom?pBz}GROT2)^k*gpKo_Y=A?^99DNzzIBdD))0n%8#QcS)CdsW&Ymtvxa% zsGe$Rm$faDUteRvVJYBCV!HU_lGYZ8RD=#MRS*HPu5?{|X7=_I_FFO3OtpY^BKI!oAX1=tf?KMfSS*g_uGW($77ppGXQoE2p91$j$t!yv^mV)HFI zP6r3=02tPrLe7z^id;*22QAtexg~fV?EPZuE=R={EEwpyQBnVjNSg+GVqh12ky!eoNPy%vAK*r)rsIo{! zc<)jU$I;DyEK(|CiDh}0sN~l)U88B^cjFUO!|97+8gDx5q?h3v9~PH3SA|UR&j?$7 zt4z*467^y)QV!7(nyO6)+#xs`e2fK78PxCzo;afafEe>Cy`NMg!GSloJr8*QPVTi+ z_xoICJ@h{}f&mqP@rUZ4X@*o>`Yr`LWlk$teiK=v;<&QOh15ZCW-oIoJ5IqWUw+oRNpOm{NHcc8OJ1p&j+}VI4Hv0i zl3uTcvW5jpq2RmeGvbiqs684x>J^QgEEK--!;@(fY>45d?`3~P3o<{1n#rG}m??^t2~Te8M#Knp!$XU76-3=V*b82W2b)0?nRqigCfH!B@4ku;BhHhQiRa)#P*#bL z`mw5c8VSs_5YlB8&EK%LMnk+tZ9`nkJCo4oxCmMwF0+;vV>KAf&`i>=6*^{2c!9QZ z%x*6x`9Ur{e45n0L#gMaj`NjPCm(n9Rnx~fx&0YKnzWX5Km`(&_AR$~x&fVqhbfwC zE@vZznig%6o`{(m61F;0<9cwYRnWyUx-wAq^)=|H&+i(8zthzw2bSIw_#Uah(kw&E zn5s(gG{MD?(DNxLKEj|J@-|eJDW^B!bn3F)i(rjGO&-u+b>|x*?W{kF89R%@K5iTW zOg=T&CH44i!Gv+bu8*Z|n~IopP`pwJSUNLP1w0(cErjmS4Hfda&G1~|q@D6L4q^&h zQsx1Bt&@9Qq8xl1OdZrXboZra4SoChbvL6^DZR&U5D@Lzr*a}NHofpw&IckEZ+s6K z?slz_f?RjRcwFdr8&HN^D~|p@*ARFx8p} ztH;pa`jL8aeZ$ zHj?_xdXpHv4}JtI^GBLYv8G@do*!vyNNQY$G~|v)#}r0atyoy$&xo`PHv7IQq!J1^ zHM4A2n5jMxjg>XtHHyLJCCLmFaFhcUjK}B&BtBVSRz0m zbpMs3-|gxpT-V(g(|^o1^ry>zPdj=po8|VPNcK^DVHNJGfIIHTb+;StxLl^LqM^HS zIB98PVPYmfF1tN*Msdx>1+JyX{O(&sg89x>H7OYfB@~Wklk-|9^p6}7(Gr{W+VZ8W zmecD!kI7yog6aLfd7^q)40`C8$W#9XBJ*d&5{c6$+t2Ew*l1r>9F?va5~f|qtb6qy z@RPzv-&C}})H7u>+Oyj{#KPZk(y~uMAFsFm3@3y#QDNrr( zgCwLSwt!*NdDJAB#|LMvKvC#+z6Y)$+Hxydm!&19a=8rN;N4w3?c-Mjkt%1mHE1O! zx^&u}ZEm_PFHX2$L$wRc9L+%qq@|UOkr*b{tC6vhti2K$2B?+dVtG5y2hMH91JUA| zOMnb50dMGNGz>9u)A3qER|C^H)G{OtQxEs4+Y0k64_rn}3|dW)$EI;cn|0cD9}O7{ za9hac&Q492Ri&0$*aCy*@&(W>)CGcWSO2j6fkT+{K1FPD6x4;>C)IH`hOK_=Wjj9D zy^sgjv^}P70-dPl*l7LlQj&Lni#+Rki(zw@0N6=UzbA6dmWsvT;4>2(V=OozS@_vU zj><#?HV2uEhQUd?n(U|gwZQ}vr|h-RQKH(Uhdb|sjdRMeev(GBS8O$*n5UHquM0ah zC~}B{Q}D-@f&&7f8!@W)h`}K!1VPd0z3u6hjCVn7EN>p{lEu`*hZs^6aFB@Oik*e6cX;k z%x)x_coRt?4h(%yC=?K{2~JR7ZU~embf>4bs6sVF(J08jzJMO(P;e+K8OYo5R0=hm z)L?V07F;tryy(fs!ZM&;jd84#4nr}kH^?v^Jv(Er zW6Mu)FWyxg=zLBm3}UZPl^aDyyU=g?3vH^rM#1A*njE)mz>HwZMrGjh;NA0?O_Kil z_D%xx<8rNrt1@MYk&O-AE*U{6LJ^ZSDj&+J8d2GjH?)i>WzOY1DfJA*Remwhj2iLF z2@ZXmjIP-9<9%bG9NpT)_T)SK(CEp)PGB|Y1f?KI!Qe*m^l)6nFziUOAQB&bclB=V zOgHruCU8h$;e%L2Cmm<;u*YJIf)Tm#xM3(54g#Y!hl~XTL?@B6Q%gatQpMca5D57{ zfZ`ouaHhgqm!c>|s6>I!FD~XH5!y<)r*BXQqXZB|%INAUuF}>Nv)mBM{jKr8WOq>-gyP`PU^?C05s%1aOZ4@)2vF^ElKs=R=bhqEMEQru-L zU>rel2|KQQf)leo*_ths>-Q(L)xrdW*wL_ovcMOmg)#t|ZTKQmlLs0%^5S-m<45Qj zW3jML=S|Twb!jc@APjkyZZRcQiMQt%7SI#L?_4$l+HwE9ra_6p$u`gj5g{^3(o^Zz zSVSS_d$&=I;FdzgKapS(tq8;YMT1pTzbgaJV>ag;eTMiz^+%`jlW8fE!YzX!&5~+h zNNLVM6ne!pc}!0r|0=MB@<-RPo%c$DeGSPV0#S&6=RpIrU={99ZF>vx#duzXSORdp zVTK`2C><3Uf`zpbap4&)zAZj(UPca2v#aBJ^}V(k-DA4d3Mr}Iu&B0 zqM{rQyXm>PU%PERXV!Z4;bGx7`)whTFnT>N07O_0E19*x_Ze*)@$pmoy>1DKu}lXY z^8p$vJI#T1zliwzduTCs_E$~^m#{^hhLlh=t3p-1%8sP$zD&B<4^Fmo#WmP@#sV> zaXKyDYFF$x$<`a=(gE$#w_nTWOCu2+cG9=2Wj1p-Y;rhp@zK^l>g}+6ZfQx-7M={6@6@GLY8lyA6;u1Q%xi6)!H+@G7YrKD{4IRg3t%+;o5W=B1yS8vN*!^6cu zsAF_|+=%<(lV9gon7G1O0Y#vgurNtU9G#XL{6KGs1VT$xqQ*=(r?NysqYOPg`}{wB z^lc*d^>tY-T=0F$T`;`^PUjdhFD{>7E;J!@RP@YUu)$63b$XUx>_DdzfiX#md z5H1PgFB12)undL}9v5nJjCh!r64=c(j|r(NK4bVc8$ z9peB!i{0%Z_|2uj_dTKgH8yT!#$T7&>Z1dZ@4@qCTgu$`#+Hkg*Z1l)j?n*Z|Gejy z-FmUuq1H%qn}%6g*PIwErtGKx5$S*ra}!3BKrfs+Wqg5TW!l}0jsr~$;PD}T;A^R3 zQ9Sfyv7uQfA(zqq`r%|I|MMpcvaJ#Or`6I81gw81_gjwyVL)`#vQ38@)S&f9cbkFl z^L^h7-)5td2&A0flna>rc8|qvCWk@K#pw>W`=x9t;lfFsMps*bM#KB*%tC<6*T`=b z98bHa)uWwOU=!ozdjTGf5+5F>us{PXFt-+YbWQqo~I&i z^_-^?OPK)|-AI=Q~w=lj|J<~%xih0D>jp{C92^)Mo2 zOF-7Dy%njpNI~_f!d4ZsqrTy?Ob+FwCU0sRDW(%tfl{{{9gIaYSKBE1J$__7Y;kdg zbbDBBcSfBh>%zmk$l?h+i|X&+5odVQ+0q3bY_EN9I??C0G@RSy3~@S*WewJnKs>To zZBSSkm&*yyjAti~UrZ!tD@MBBZ&T4;$R8g`ZPCNwYji+t{`UU6sv!BOZAnZfoq)@G z)q8;6N^pK^>LzQw&uz-fkX}QtfB8N5tz`)-9F71B9o-JLX0NB+#0P70z@hK>UB9Pp z$1Rj=SOmskyRXMWyW<(3I}B7fh1vuBy{2#=`O>CUshIMivcGV^$7xNRci zcsqU(oGvQx|8{LzJss1({kZny_GA)?LBjJp+r#H+9gB8(j(!P1#N!P-xSL^+D5GbO zpA+wUYA5P$6vxh-sMklo&VpP`zb0L67O5lb%gaL9ZRb~|N}m6&u(!G3g(C*E&wyAl8SGp~2CR;wDZ??g@W{ikzDfA9PrrUqcD5*&BJ5YB8M<83TqH$D&=~?p?UT~W-cK*}y9t#m^xam(wVHjL z;-eqN-5va-r;iCEXBch;cH}f&TqFHq5fY79R?3<8dB&MN@H1E3R>Ra-$VPcgS2YAE z)3mW_TlGZrH>xw5MjDF0RsQ}KTK8fg4`KV6K<5-rtB+`V+g>_KwGMw$O-tQ=BbUdG{*#=2 z1_`irw*~I7s`ppqdq?l#wC`f?s=Eo9z0Xql z3sniv1^QnYvDJ+}HPt{JkC;4YDT4lTiRtHSn7*a^%ezy_p!W<)C=*kzVDNHntKo;X z`g>D8*AA&d#x}HlZ0TXAb?Bm*R_OVmsSV$y-1(Udo>UuFqfET=edsAVp-mxwrE>7L z_i?_}YGrd~VF_v-KO!IEr20SYVX&YfN#JrUcccRa;gM3wPEyD%Af~yp!dDH}LJp3 z_}j`-9Lt(PEU~H@LX7<1jE+5A}>+h+Oc7^3|tdSRNvElphGyq;6-ND}HF z0Hoz~a%e4ZGC9Ag-*au#_b>nf)v%z>+_mV={|&zyF$l7-9!)J4yFx3UcQMCp6A~nK z^=57;oeFEXvSfBL5^)%@j4G)r1I%}p3$_j25q=JOCwYVbr7YlIgttZRZG!8vRnC6qgOZQC zA=7H%v?>DQo6*$jWnE!vvN}vyDNF0u_p64eqp0%8V98Wg^5QRbPGsQ+s^0Ft2g*!l zq;_~1vPAeKo9phk^nq{6A^*D+F(MFHVeC~|GHZ1hEr1Fs?-IAvV2mTN4kYJ{P}_Wy zq^P%&6>X($^nov6wlvg>81*KRZ7hP&KQj(h^m`A`*{;NZR63~(_?`=>i+dx7e>?o3 z(JnL?OqKo*kpRF0U(D8on4O#q+y}5ePk(P{d<~nQ@IHj+*bSVA!W0h0vz{3WXe5m| z_&i|~IyPr%VMLR_)7p-w)eT_s?+c3*p zIiUtimx=!|P))HqI~FnRcV2{YFO>LeE1AtU__ep)ez$RZ+3lUEQqhvEr1 z6(9TCg~;jvz%=CT>+3ufp4b@{wki)(__6|~6Pc@8cnKf30xuK1y!O76pO4&Bw)>*E z_80NHs>!@9R(2cwc-PHRK6_E|Yq{TeshDyY+=vKHiTOpsPd@NA=b)-Kii6ALTneM3 zS0Rs&zK7b@;C8j%nZ{g59LZ9yEio!2j;+Lmfx;PX=c5{2mO3Ve;<4)#fzzrcvRke% zp~^K>fn4q1^A~o9#h*^M+4ZD)%^p^hg_LBt!_*XSZuZ0c8vNfvD=KzFgo=}#Wl z+<4u#L@E*Q>=}dMkrOza!C;WpZTgt{^vo<(GSK@j(>EfE5?Bqcha%w$|i+AUfMRK?t*ZXxcK<-4RMBLbS z7id)OwE1W&*OA53-fv6k1@7xRy`9kIRG=tuZ5J+`&gC#aDx-?%MX?EGI?IPL`fga^ zoBSmYp|@WtM@M#Ivem44NEE-D{!Z_(Qtfp%k6%(!Y-fm$gQ3wPep4FSn$LIi6Lqf1 z*iyb;2RD&xl98*c|7~$r&+*!LF7w>m&8EWbc09>eANM4e6yfecuHqgf`?s=eTqe-| zKX#ryaH^=jd>cWF`v$S)l$+ww8~B#Rg^V)C%$*%Ko2%@_dZpQP50E^TQxs-oB{{M* zS5MlyPv3{^vuqsw}K_Xfxy0_b`RaOPK|JWU$p|a8C^Y~o?kO!UX%arz^}(wKf#(_D!Mb3RwXQi9BJhPi?>=w`{bQ7r9@$RHjjChEQz zq-&QkcoQ*;KO-}1Y=QL53LdhuNl-fkMm}zx#R&YwSF*>T9t69?un$mOpGLCmLc{n1==~*>vA*uSy z_UY+jcD5v{&05S^vaQbI`U9OCvy#&+yTEA5bpk+Hp(;6J{6nZ;x$KHtQ5aNWUXObM|RqC`npI0_2`Vo$T4ZVqr3P%%EeG?E_W-!RD#`5 zxt48RH+tsUMj~9Gge3GofeaQS2u8oM)6nVnRUE2L!XBf4oiYtjef=kFafSpU8D%wq z!F&Rv9(Oj5)I;WI>HT`lTNlN~Z;1`-3#$gB)D18y;y1=k)7`7(rJCI3R%S+n(?j*y zK!(l;rwvA0`=1O43=A;mx(t z19_OWVvTkvBD@>Lg9lKrFj6U9X^%XAIjaLc%WN%(X$e664cXMUZ@bP2z=_4ILmoBt zjla*vaqLUN6Z$4N?9Y6egT%e`-f$52Vo!+O2pt#zL zrgJ4!KO)DTE9yt9@0+u`@vu=SoyAy=tTCRWGzJeJ^0;M}>xJ^O=f3j{ovr)Zz}KIF zy`H@M(L0Q5NvYQdV83_9i%c@O@4jOPObTfO6DDnapYK5GDeduV7rR=7`WT!___NZCHxp%sQe6(?(U7OCrl|@QK?OpR1L$RJcZCwNJS(d~ z=Ao2_szX;rWLMg#;0mjOaH%D4LS0t>p_i`zZ{HOt0Ob8$6G-4Vyq_%DC6Ph>fUy}( zpnBZeQPa?`{njXGp6dr;Qp6@4-HnkprQzP)$RKf&(rE8n#IF7caCcl&)2ISlE$?-% zRqmFT`pnfSqW)R6&$Xa$t_4Nue13*LGV>V`J?moo1sUaFdTj z?9T)Q^|PEWOsf(cn-odYhTXfZm!tWoh(Tvi2-A(v!>Zf89#5CRHM&gcs@Q~O%e&eC z?=-K-KMfe_Q1=U%JK}wuWw|4F z-7J^GE&Dj6sBRN92CE8eOr1E8QuDn7{C${Vz+}*`T@~tgMyE~{TcZ?xmBx`!p?drm zP1y+ltgLy^`2RPEq$&0bV{uTE4`L4$HzB($p4tJWXM7yIt zXca&K8(|`N`7U^?)e2QPx}!2$hQ^!wxEd?o4&8IEQZIZ}+-Q5d*$mdoGI<+t3;We( zyOoTO!&y$U)?%SxXYcmIVG8}f_jqFzkkp|ZTwLp{P2*BamEDxQvIClF=>Dv>L4l8) zn2=zq2(2BClYdve>*;3v@7KGpb+u(69cpj9ufurp`|w*UpvKWIWrBN+ltd$ zYe2cu|M%9Ujf14!`mDN!)%|GHPlp7r5{Dn3e(%3IA|k~*Za4Y;uQf{SJG-2IQk=#; zD8>_Ty-!Kp!sEMhMRcH{W%A3#;ReYpcYE|zDlp`!OhIdr0!QQ;b~G)Okn$Ejhmg@0 z1~wg2C5eFp3@;(si_B#*QD~azwu>L3SjJ!g(!@W(UJu3EDKRGqLrdAjvXYeN51>bn zr`=|yS4`$H1cCF`_+}dmobdC8)X)2+kr?Y})5Gvb%GLUw<4ByJcUe|JmPCLRD@}3< z-v-~OsI|7X!lO8wcZGcZ$KS7={W^(+bgo{)KyWMho9N$nU=a~^7H8uQpt9_LXi>#>`_qQ*0 z89VC^B{cz`7%k8$U=lTFoeoZ?6idYvgUy<9t(iP6G_h7+L z4Efu4L`NeTX)Q|i6CphT7+EXlc=d0=Td>TiY&G>YJjD}JQs-tHwH4uYB6rYLPv#ut zIwZl+(+Bujt7(+vSdzCdTX4K?q~Br{rtG+>haz^K-mJnSczzH@j6C_NcvcU{qIuFB z8XbmyAG*qqvtF;!UqK?ir{;R^tW914Hk#aS*3!~2u$q>%h$ZJ_VDcSTuteUO@x_5Sk>nE9z%;(y0ArvyOvd06J1*{EI(@)QX|b7`Fsm?p z7}xot&I(4Zr(E6eFjP zi#q@BR5Xa7x%r08aUVfs$>F&`>5&goCjf;^rrbCrn40IFVKNQOohc95*XLjW7*t}f;n$;%BB)U~AX0*V@1 zILGxwO7XIa_H6VV)Hp@3L=S5JU409ikpsbU%F2&0qwOAnDvl}P7%W5YpfC{v!)Tr9 z6q58$JOTv9*FQhL;@!|5FZf<|N4!oKiAfxZMF9(ZK*6t(^peP3RUOZ%FAHe>0C$5k z+N4>YW&975q-D2qk&Ubr$)x}L0;>Rp^?RNxa^E%TM}fG5cp_fJW1|654FxxzxO^(_ zuKvHQ(*v=Xb4%_l2nf~ozyAfW*#yj!FH&YD2z+uv7ehM(MT)x4@kY>Z)aV9xhY^J8 zB-cZM2wZHHPjS%yEzKb7V2~BlzTl&`^i-#*uD_Sp(J^Na@77%4d2X36iXD|cm8kE@ z8>3nKI3q{OMb=9|2RB7IY2l{5jJwu;1xgFu0F6+-XY=xb`w%QIqM^DYOe_pozQGV{ z*gRALxd9XVUh1G~y|~ebk5L`4^o(ShY@;pV)>=5GF=5pPUHQ%ZlI8r}=*Yr?D(GBh z_y?q-=ZBzEgv@7kwxIbTKu%s>pV7$pErmjdx1HztH|4M}2=K4nR<9T! z$7IH!C#8W&FvDeEQ`4H6;uq`mbyl$9%i)#HCCb|E{d<9x`z7KnBHlEcBf8=&|Npi3 zmQhu;(Yv=G-QAs=?(RlPP#S5FF6r*>M!H41QBp|(>6A|CkdT&hOUpI-k$S zJ;q|O_FDJ4=bYEP=5LF%VZLef&~b_7kBd2fCcmesq!>p!tQpStu}E8k&?7pY6`i`i zA)(C~n_>jZ9H)YToTUWol!`oE>0qwbBGD3$GglH&-ih)@;K)6bosP6e;%w%Av>$$3D2 zF479bkMk`_D_|g4!i;OZ8yRb25cqoB`)iYAyZpOy>D5mJ0pEQW6&2o~AwP3Q0<0j{ zgZVmU1O_V3?&~Fbi%V(BbqvHb|H493_NvU5Z-cF0l6()feI$ck&(ujtXxe<3*6 zjm@kBwnmg}uNc`^T)d;|Lzu4;PN+p)aHr^4`g_?0J{m7jnXEYriU}>^`;6~N(DAa4 zT6teU%_nRSj6$>i;J4fiysat2D0r+*$^5sg2Vp?{kRu2|S8iqg@5Nmc0~Omtc**p? zofsqJiFvUumCTC#RM1f%T%$u#N5n6A1Mq=&tDZYKsKmV5Y$pp%2ViWq<>%iJ zRxSdb=FA_=VboTtTGN3zsc^K!ti(j8!+DN_?e_L|a&pb5ji}z<-tO*hnP|L{l9ImO-e=J1z-qNrrN@UV@8ndcau71Q z1g6q1_NI&#tUC|uht+8TDIu5Bd8|8l%TkNf^J;r zzQ6dy4Gm@m`oIi+dREqAwSJq3_iy35!-j!)@^A#Po2}&RY!VGWkP(B)J{Fb(-f)Fn z|39AcPq76O2i$S5BqX2=pA-S9mMYz*sJOU4C;pGaTDnep!DdL{Zq-tN<;eU};dT!Z zsl%x;Mc(2Xi(4pqjrtS$*s$ zc35inJTKR%c;b8@uYe&F;3OmAwawU&k4~TgUMdO*Cg21N4bbJdNl-8{?k6d-e$=Wl z7|G-z=CfZaQA|%|(h?+z8y|Oe;I}%^N+){L`3DeOg5lAt^;%eQO^EE4TQ7mwp^9n+ z?aI!}%gexkD7aO_ThcmBVfb`@Z47=sB^br{lDr^sE10!*A|C7A`)}Xmncv@5DAcxF znDiGj4mHGgmq^l*;o+55n9n5CNQMw$0Bt}b3cRcdqWpWmJrQb29RV;#&<2=yas*vB znA|1+!=n7&d^pXD>m#dxAX5DUi{FhLwjh+PCpdv%zmzT99j1{4QSYEc4>NUzRB=47Q~ zFk~KwI>~Wj=ek^d5FCC9a5sJ+w;|@WMW`w*E?%>bD*REQRnzrw3R>S?w`HhIDO#)hy{{1gJ?63`F;^76tn|>xc+5r(q!(*VxnKdT#SY#Q zW@-XjN=I5vBQ%if?l$Aa&|K@N$-IKUyG(gRZLke37jzULBz@6FTK}54)nhTMACN$z zso5kbZ>+7#g2z~2%=laJi;u@R#UvO%l30rN% zyT+}}rdM#j-SPS^6EX<{862iyhE)wE%%Ftv!?S(TpX0i;TZ@7R?#^52aC`~v{4@Uiwg;93gfqS1unrmRV={~l%{ zO@TWfl(T1Bg-k-#4qAyREYk=2DIcCGY=KtadyzG^=gL^!bUZvw!njeDV8@5~2YM;5 zM4!j{!5D5e-J6s2;HeF}F6mZd%7sFL2h+)Gx1c^MN!m&=9R8h*LeisdC^wt?DL?^P z^C`B#87pT%)j{f)%~MI3DYJJuQZ9X|Vle zKZEP|ga})nWo8HkPf*A12AtMCL6tag4x$!gMkn&?_IqtP*8$9X+tfjWaQ+-Af*teDYS;@cu9(Ni9gem-ht$(pm{8CnY z7EJP-r?&l5r~eNJMAUkT8GaV}Puhlh0$zAQ$^TDyR(F6gyOX~3>$Es#>r`b z?l5?Bco{|-H=d_e6%iF>Ki6rhmUu-_+}aJs3s4W{as^%M73awM`eX!GZK$}JeQf$m zd&1FNPbQeSnWu}B%SXp+YdgH%92PSZXB*tj%u<$RTWnV{JTK(0$V5mBc!@-&Tr43@ zKR>OG>bE*qhZ;{^FKu|X*&pr4<+Nf{j?&bVjs(8!Pg9mdBi~P)9k$opQG92b}W)qJvy%nG3va=pl}MTbt^&+q!k|JZfxxj4YHv#@^t7E2mdSSWkJ`GJmgrvKQ|Ux zlXH)M(8u^csP=;)_3wmJZ%4ABYGJk;kZp}6BvcFxrb=T&5n?IRgSKVBmu^RZ3-wVm zUXR?QuOSd+=HTdb3IFlh-@uG8d^6AAUzAbMdUs8t1Q!8!=DhvP!p4m8ac|%RA7=}l zSa3d#EFg``;W#Ec+f<6JgN8AQJy*6^vH{_Eq0SW)+4$(6%E;R8j!;^^jb{AzH840n zEeP)Exu(qE_4?^X8dr<4gV@nA=>>>M6OC5k)Vy?Op&i~+t!^p@Kc-Zpsz-ALLb1cb z4Fw@oLEqeXTZg0u2L&a??ks6dME*$qaA!_YctP-1^v2e0*CGywz^k+3CD zmEnzk?6SljF8Z(CF-##x+iK&)w9&!BLS2DH-a1gSu+2#(n3ODFc+&p-_q3f!z7%qZ z=7qkToE%5`%NHMOoLzzv6HPOQ78K&uUDVZkQs7$Kjj2Rg(Qs)>Bz%3f^9zisbVnQD zh`Gzg22`c-qobos)Vyp&va}2@Gt2Y_JntReb+k4gw-|J^%Kw^6Q-6afib`BMZd^m^ z%a2ul9(4A9bAh36xj|zbhj5j6cnGY2m zU>O2+XUSM(O2Rc9c|M+nh`AxkWj$R6+C!xLjsee^4l{=tH~orZ_d~ycDT8=&u|Ls< z_mYbplrm+BdmE4e3dGU6%4 zOWESC1Ib;Z-J&K`v-vO{BPBU3PDgAvBtKo>jWG2|=aPbbCD?l(h0DZizm$=YAx{_X za<;h6v;p!oa(+jn4zJ4#sr)>^UomWRvokQr9ZivbMnyqn{7f`u=m1Mh7Y^C4CkDqr zua69n5W<0MBmy!?R3dI8gi}T;SQw~P3{}~C&_h8bkqBtZ8>fMWR8MpUC@pVIUUvds%pS-A1!@Mie%KjsWNZs$Rtg z^D-dr{kg31e>iLyD-))s-UI_6gg`wLRhR}S95$(_l*g^)myqww82t}BVvmCw*BN*( zoaNsSha~wKIFIne3!fo~ai=S`TD|!Hg;V}Wc0q#b@4q%b2*0+yknG2NyP+}bVD)K% zTD3a_WMt}$IhSq#6BaoMcfkDS(`lU0F#)gbeAAr4Fwg{+`2Q4UHvu^vT)uW`P`q71f?sa@Y}c3^u|wh8!eqc1`No2 z&|_xY-*SZ9IWddX%3FcVbG@F=wjtDCE&)hp$L;4TDQf!kh_^4u9cQ=0H)kgnbhRfn zA^@|e0^>fwa^e0;z&~91&P&4yQlb4uuI_rrRtkQGs(AB<_ut-mBZGg z>;u6?a4NXmu8_i7?QYK@(ZLxYq(wq!(>)xutiD^)PG{AJ1Ya@KbPV6e*UZa|^D_iD z1w+rPVXEV)K~QgdU!&O2m#TQlcv3Mk>oucf_D&o=VFk~ZM8oUtzuFlOR$@Ixe60EB z6>2@?j1_d(>Gk5>#Th*4#$Ug9Ma@+4jV^~1$kd6qE)kY0YTmJ{n@rB7%bU)JbQBgA zQm~}PSC?W{au%)|GxwE8qBA1grqyf^D@D@4 zloqY0&aDhazt{DF22v{Q0*6AtZaGfDT&7;AF6Adp22+(PFUXFGF=c5anm%-Z+qJ&Z zfjGUF4eR}f9sezY5ku%Y-1C%Nd+)TQilBfog#33S*#hV2{@2eJ>BnzQkEBa_9$<=S z)ARY7X^Trrj^^v0w`8t99J@m-W^-_TWROD&PygEb3N*51I1&mw3Tz5t{eaD{lHWwM zZ#j${THOMdUO8@G_%d7A6KjmqMLOe?VVB>$JV(Fi?QRiHZ5p$0gL-#Tado*1)ZoW= zV0W#!$5c0f&z^``_HASs^qM$A!S9=BINsG9(zVmEq9u^+wady&%HSW3>{1r6aH1`- zKGj*WpF;W6YIQCt zIA0&*1gMH^-EP`UCM6Ew)T zqr`c{55A8Mq9OAUq2_QM-1{iKYVMZqmXW8-GIy@&pzbXw{Vo^(Rz}uyPMN2(F(ZA4 z+mTc#jTD!(>RcI%+v{waK}qku&VbLjy_F!}+-Cu1dcHXlNO8UMy9CMExM@Nk&W8G) zo{n(kmX%KILM&p6ksn^H`vn1G6W^Jc1az3(vr1)}rq;CGnyW0A%bO2LcA#^s32kV$ z1~y0p=2`Yps4jyV!@uNI6+5!_GG$S1svr(!(;-#8_?e0j2}dDamS^>h4)FU<3$D8q zPYK1Yt)_M|P>M)GTz1EeFG_x4e=x%2JVD+3HcBdlPAYnKbbS(qWCx=h7ZJ6T(23cA z8-$sd*w+u0PF+ZqR-@f#^sv13F|sM?khj(IJnir>yKYqlsi$rElNv0-or-4Fed5K% zduLv|9}?Qm*!cvGb5(j$gYO%5j#j)?Z!A4$_pE=>rZGGHc0QYa zKX`fYG%1_aVjC(t5SN9(;B{2Y`bFx-YR394?5^#S)534d+Zn9PGMsY}3Gwf-Vkvn?2?H3aLsphI?wuTse8+9)sjc5`1935Ja@ zyY35AR5tT`mA9^skK~@I;L;Fvs^`V-exCpSWRbN{ zhRCx^2wiE|>nrv&8FsNQ3NYu&!g<1rzx7tVh~QmOk*O_Dd4M>H4hh6^;d8m!?2hxR z2@d86qV9^;{pX_lyqH^`jeZ1&q27Ij@wDK7|06O!2fd>&;md%P;#M>v=Z-GK6N>+_ z_eOl18bKWis9>qk4ct%VBPjha$`A-XtH4~EGqhk zsfAY5z(B5V+jaKdhx5I~u!w4^lGD0NYKhasDU_BwRTnADbrqdXzuUbJB-KJ5r$#1i zjxcl+5{PMT+o@XnNrBmG{Hp*r6KtVOA31Tp7K~n6S_=0roNc}n5z_!8-tVOI;T)g& zUEC#{-Y3F^kJ{6SZAE;C)lOG1 zVbl{*V#Taz*?8f4pjoNyHqw+$gKOF{Q4)=JxYA(*%m&h_Tn_5X_C)th(v5G+#Da_O3 z;$_q2f84jJ2DJbea1o?4XLdQ9kip>66E5X7AfTHc?dQ!^8t|O866YkE9$m~boHzzd zmZYO7DU|nl{sIZzDS!hne|EN*X6VLeHtfuHrs2nb4bMV73rQqbFSh;QF&KW0l28|g z=EWOd+vvHQ!kAJv>XMbO62$-;rwx-*fN1jq$#JHH@h9pt^AVafK3|G)NIglBJp5Ca zaUObVp{$q*4ahs1B+XFj=fB$9M+o(V8f^~FyxkUMf;?Ds8zBOpS0Zd=o#J8_IqkW6 zx~ViD+xNeG4FM z4%jv~tQYoK_;S;|vk7V+?l4Q(% z#%Z%3Gs*kX+M-e&+7Wv-bro6@Ua4LIRc0`F@>{sq9?A}f8hz|Lc38Prj)kGKH@nZ| zctrkOZ|;X6jn-UG<^7S?&@5H28eV05648gqCCps{UU)hgJLz6k3%b?rXd*hMYFB8k zii#ZSppa6^8$-92Ik1ggwi&pFC}kwX(ApY3c3QGmYl--{hg9m{VLyL!VjyBij(Lo5 zvtMBr)&TWzyFj!%<`ORMGT=-AfDH#r1-|g;zAm4)Agto$iaT*uB9m`eQ*UN4Io)ja z;l=}>v(_(b$bIYVShT8h*Nva#C>^#)vAA)~yDJ^uz>21G zom#%8U%p{v?I#K-c%0EGwR@mcpwKe$be2BFH}rvelyG&d?WEVl%k?=M1b*QP5*3lH zc6wup*q7q8hbV$jGfsOk`Y7(~OWv|$My*J;qk z>*Bm-q0LJG@EnI&Z|Z|=&Vx)^m-);Ir6G?#7hS*R)R2KzPbtndd;K{Kp@E3Y;8BUC z1^*d_(m?ZVB!b<9y#M`tM4a$$8#y z(ny|Ff7(DAjrf^Dt1=6wlpQE#0V~= zxmIyYqwN!E33TBP;NkS*8+fd5M6-dRJ)z%QfUk^yiPnM-NaSRmA+jIse$<<50}*(a z5>17W(+mNrP!uUK^JA2AzWkRTkVF^loQ(?NROvMeqdPfVe*N^^d71og)v4t%a3IBk zhiAuR-E2Mi@smlr6_vIdqtw}!7DJ0?KIv6xDg`T7vsr42Lu{Jd5TA}-ODS^tBUyBG zj?ypYg@0o-p?ws)P&8Gl+3D7M)CfW~~nSELl9=u)S=&|vPA z?9jxU_I;~{O`B@)H7I4`CJ6xoBL4GzCv`WA3>-Ftc78s-^r;R)QKihC2=%u^BrE!u z(w37@49>P3ljoilDTVStHrDI`4jGWzt&AzQ!6L6gJ5hnbe$uwxU%LMf`@IiuRm=B!)8z8HKcnYdnj!bV0*^U4NF^ z;qx}0QYj>^J4n+JnuUePIA0`j=m1?zy&v-)ww8H4^%Qgu_Cp`)1q1z|LP3dMwDfz3 zk?tlaz!2lhegIuKHHsVZZ$+ahX;<%!NP2e3jk@li-Tr^h^FRl0o9C@7;p|pZND22g zdruL~AwpmWFbjQ}{NjVN#w7i1btl&w&4H-hT5S#zLm{iO>^%9cyk`3O;;-j{3Qm)O zuoH5ay(e&;jcL;x7f~`HVBTg!S%I!kRSC#Mn)>2cflrp|HR0%?V?+Zt0Hnr`riwxH znBp+}y%Zc1gVO%|xbx0_7h?yyA67)k11|+skPZk&@7qBqFuxT@2a+_`xod@mr8d_7 zWG6Hs#VB{}gDGFF%k_)E2EPnkl$rhK*eWG7w(y3MevT_ZNI3vh&_astpu-gT7cT$Y4f%sveHud!`(!@RmV^a%h6czdJAe@_up*1H3!bw%?QgsE%cSAPm zO4sof1j4Da@@UvXnzWY}aiJpmxGKerd9F~`N*X0XrgrK)BZii#%cN1T8l}}b>50;y z?=DS1fnI3c8fYKxFOXHDeZJ;?@EQA4(q>lp>t`#D(Bm7!?cStm^s0& z@-mm?E<)ZL3$YizT5*1{TAiEIc}J)JP*M&9Pf79vHFp1VSw3TdxN zeG*E@Xt!~ixmX}C!vzYdxu<{QNZZ|f(ARs|f*W(gt$}#QGD_@tQn@9l#fRDJ<|P7x zoPs!r-B{w)w{!$Z=g4Okqq&=Ii#3Xm2(7-SYm|7}bbfp@$9vd|?uYyyyW=5|$##8R zKV1q?_x1+j%;U4JebuFD0@d_e)n~PsTbVSgO&W7J&1I8*VPcf?5-m4+B8#4G(kanR zW_lJW2y-*v511>;k7b_s#OdEfeIHGvXr0^Fs7S~Za`^g{T1V!sM9ZGF*DakJT1+1Q z`}0U!*Hq>8;WX9{eGF4?d6ldz|Gcqu8OHnY@@LCt7cy#7!SXLv%_@W7l83gzDa5h2 z6)hKBd?%lhMeX_zBvgUpe+=B!TNHhE#QuQ+H_KH~PR=D7FV5d%KpA7^<*hc0u4ABw zg5PE%^Ki1aC;LDopde{RXd-<>CgMdmqY#A>kI>|dg55sZtW0cy^K(8^lzYg;BKUT2 zJzK>1+_oqw3$+jV5ey~{SV2$<*-epj%MH7rUb!Y8gocL^=7&8HgxK8c~~F z5tP2-U3NTGQTIuGlOlCiW*@ayglQrCa(eT9rsC*{;*Sc#1&uu|QY3N_YsLlkUBRIju`dj|#SjT?4ARyXi z^E+*XAY$_;jTE=1{5>%Rg{Seo^Ey@9|!)59-%ae#1f$>i}vRF~nKBdK!GcVh^tn!8tW=J}!KwCm1ch2964C{H=uBloT8^Ja!ju zo{u_1f9r15CWHuC?WSujr-OMJ!1c1ksumOy%F3Syo)p0S2CfCn1cEcJogra4evtkpABexPENjHfur_+?obIgY@i$jEZ!% Iq*2iS0;E;8LjV8( literal 0 HcmV?d00001 diff --git a/docs/images/arch4.png b/docs/images/arch4.png new file mode 100644 index 0000000000000000000000000000000000000000..bd055feaf0b1c96389cdee717016cde66a1c0160 GIT binary patch literal 39566 zcmX_|1yCGaw5{>rFu1$BOK^90f_rca?hFKXC&Aqb?(T%(5+t|<^*G`%2CzQg)y$}A1<3??Lqf~i`uJiitUULgKV&#a1wRodF2fI$UGkpK}4#``0J-)g`!cD5dSLgU-~8n)6_ z_-pOE^;9OOwE?SP`z{5km?%0r3=E`LuO7(RZ81)v2Nv>wP6ZplfZ_Lq{`;jUy0a~C z^xfy7_&Owrj0_$gN*sKJ3J(Pwn7~4nmzN(M9cf#%Cf$f{9E|hM1^&tr%1%r)CCEO1 z9*V~OEZS`oNrwSkBNd{;fdpLDK+g7gwkYuZbt&BFR~Gl*+MYKCrXA~C|7PF&v;G|0 zOS{FY$E~lVjkOqbI52`y#aQ%`|Fx4=jEtV1o{nxbe5$0VC}Zr)_8uw?nJUr+(M3w~z1}4}wqPO0BB;fUS>H6T;QkCBI;wPuu>+8bg=lk>b*ZV(e`aHJt zhx^%1cUM=gZf*jeXKN}VuUOdFTD3-9YK78TWvam`Ur~v(gnW4<7b>(W$tA)QVr0i9 zLnLYCslv)qkZ@V-|8)DOGHA&q}+V}_~m%Lqa|vs&F!Gm`?~D%>2K}*?QOMob(S20z8c0fI{HExT-bjP z#T#U@dLGNW5fKwZB;EEEDRn}Ki?Oj>0s42CQYOGI&U z`xf(&xbyDkI9c6<6xAsd-nF2+tApG?8jKl-mxs&z{QRPr=;-Im5mIbWe^sgScM3>S zdEIvRa>U9WG`wfA2;}0~ND* zVrA+!8EZSmhu>bGHLf&NguYyCiZMsSAWDc*_4g+vdI9~&nF5AQT#V0mJCZix<#?&K z$8}HsBUXkudNQ6Sm*q?t5fN{juuNjwy;eI($!m zH@#oyzOMy5pK@AF+AP;?F}JLR5P7eES^AkPBEre3k(%etF*ZJ~o;aAy%g*ldc(rHj zcPUjYpX%lQ^5EiuL+X3No|ENQmbge^GaG)qP}%cxH8JoN?JMdBR3dn!*=8E0tln1r zEt-|RN)kwNaUlW>9N_NLO68diMcuEt{pvXHjy7ui@^{f0K~yx>^N&9-gM2>-1Og-E zT}x6{P(VXLVHri3m8#?lyY9jxf8eC1r7irn*5X7eSfN>!lan)T?cu?5B$ww}KHFFe zqgpI2BNO<1+EPbmOpgw=h*1fFA{rR&gQKPunGq4h&(EJg_k+p3?6-8RyX1n6dWavl zm}tZ_4EiGjFz}`#Ai$7eWC!)vBO8RdXd*A@3?U|1MBv0<=s1b1n|Rpe2p$QTYhxNx zVDRhfYf~vhy~juKVztDJ;$baKF^gCN?&TOTJi;~ZXZ7Owx1{fI(`>0ZUS_gKr>9Ds zjk6I;w)0o=d7>~D@pYoQRO-O6`-TL$+lFiG2O_3)C9II864ZrBM7k}3GFz%KJnwlI zHr4$aA0B5mj>D{9M>|47-cRu|NJH@rEXw0~W{`>zjn|(*XPnP&FR7&_S%E=^i7HVY zb)1Sp_w`qSRJ7uhW5D!9Vg17EF}B1gS&(WGbqm^tjeRjZHxfdZjN{%gxD z9aNbkkHc>+vqAVme5&tcn+fnRaB$+xC#T#Fzp?g;INCbzH++o)UvGg9FU-g1wc6~E zktD9B8a>QX_g!5%4xckT#eQC_xJa^fcc#vCfX8VgW(e_EyUKi-RWlOw@OJ+G7Eg+L zGD<>11V<%U;H)TDe$%vMWViBz`bq2Xv3b>T4qAFaUb<-cUq1yVt1tt#t*cam^VS{I z7|Wc1r_WDNOJ>g!MZz6mB*|W_o<@4MFiAuxD|FqS03kTolhbTD%6OPyj$jB{(z}xt z=z9}T)EM&1;LYeTVRIoJ+u!+e=vq-7I$1Sw?P~p0WmCIx9g@VP>o9x7q$CS?k0Nr& z;&NyV8zufp2akdWv>Qg*mFt|7H0>@5%+@dmx1K~O!JRD0nFikZL0T$qQ#3Lbz zd%zaSNC5)B{jtzu_q*MuyLC*(y9(X{4l#Gpzbg(UBP$>5VjX65}W*1R&) zg6_E5%pBNaztWFZ?tQ)gI|S?W!{UNHL*1_CsTD49HL7XS<$y-w;5UA^@La|oUhL$w zG-;|H{Fs$_l`O9c6N=6BaOntAY~uxCk<}|ELA|dSweA8H=x;TjTH>I!jQwx1O+DyU zcP==sFv<+-e-iqi>vDyg7CPMKTskcsSQECw>d+6&RYCFMhJpMyZMV5jxoAvi|&FZ+Us@m(R@PCh8r(VT+`-m~!mWm_LPM zGv0BOY4O)ZtfO?KK4DOO`~RP}sNWX#RD1noG{9h#zsy>Lf&&V$os7iJ@MiRw#E>DMf^=Vt~1yk}0jj2q*AsO^r`fk3D@oJbbU4 zJ^30ifl7Quwe%hhHi)AZde{uD`oJ$ZC}y}EhE3XWUxn{|M5~xF-R0{!_0pfBBK%0j z-C)@B<*#iJP7Q3Z5Hf7ZJmunSENyl0Ksk{CvG{Fl9=29aMRzBfLD`BFSz7M3T0*d^uyRygJ zV|2!Vn#gNK7Y!Q-7s7qe~n4r^Dzc0fg?2yyFN93~t0X6+G_r zLnfZ&f@^P72AJRE;UV`w2vci%c{6JxM-jPOYn`-N6u z)~J`PMwmsoC)27(Z+Smln9fu3MfZ=uz%9zgPUFFDbBA)Oqtj}Br6$}%kK9lfAXY?( z@EjayrBhhhD#SUafK>mEU{Oa|9k$hY0$=XT<_%(Rl1<7d;~r!Dcc(ja=EK)?Or z5gmfIX>T*?-WU@f+C&AS>^~q}e(4K=WrF5)T$AOX;#o%WR4rpP_Bo;+jkmB-P|gv2 zoaFpR9b`CIW0Je4Ao*R1sh$H+5ApF=$FD3%m9B5Ay{ zEkCFOkI3Lv^tQaQ=d%v9-xBxmU-BOZMX`d>ve~gJzJ2O#)*w#n&47JY@jwJk@m+%# zxYI}LKoJXc21CXtHhiafOh4&zA%=4!Jc~xUu>{naY^LXq2^C{C`&Q@&WtQVA=yOB6 z(3!C&C>^6r3qeN-cMYUt_{jbq2>;YAnTavX?gQBWl2=C{+rv3>HsW|d*5p_b<0JwD z!${I8k|-QP#wCG5z->2Vcb;ApBOzu`pQHaix=V0)FFlb`f%GL&SXWg!(B$oSEARCE*7a!1W!d1y37#!Yb{z|6X5|o-8Q- z3yV~rIdeC-HCACEV#mu-Y)f9@j7rVucT?KtGTOaFmV$`7<8mG3K}&1d0)>{@kDb-^ zK{aE0d#0Sj+eF{HlPv;{PPpv@?Mf-w3n-OLpRX-ze6gRzUWby1r;0pl(LFDxYlRY>%m;vqRpE&kg-HuAbE9n75Q%;IS zCcC+`&4x^|1x3Ks7;UD+0=FpxhLPWKd1(d&R&P+BLKpV0LvT{|a*brY_PoEc{?NQXThGULG`C6$Q!`=mTjn4P zcseRoj8?Rs99>)B4t#t0O5!6Mdq2x z@xLcFW!v&_l$P(1^PT#}A45DpI_I$3%+6xi;Q@=nX0g&IT2_ll%7HN^F8L|U*^R+m zW{XJ-9TOP!M#EvZ=triDqyZ1X{dyf`CkC9|&RgQrFJZVws_YTrjma$`7BL-0&YB2O z5BD6PA1p!kpFgxCoq3+l6n?->p=8C}*V69%N&KZ!ySiY;;df)XMrm9{##uhZ7WVl{ zLuq??+#d*53cZh>XGAzSB}oo{x{1Asg0qs;V#-D-W#Y(e=BO$3;9S{1-Y;L;Or$Ya z;g;#gSB+By0!v_TAIK!f>`QIhT?G6jgTcwKq3JjYU0^w8S6XMQzMW#6vXU(Sgd_mqvr4i9e;ZWz7|7Njn+tmiU^csw4va z_;j^}RU8hcNeORM zp8y&qqe}UBw5p_JJqM>}e)kPe8@L>0f}h#^(7;=)Tu2@W5YJQ7jXk@3PaPw)=^}iMBNlK}@2* z_&}pXQLoE~+pw6E0bk#o?-r=jzgptQ-~S0D-D6e2)(!zGdTORo!e3XWuTQt{kG}$K zypGG244Zc5dePOfD@-aEi8RxGLh$OdTpb_37t9dnNjkr37aRcsU7R+w5Mp@UA&0ITaWU#8Zn+ZdFBPTubd z7(BXv@@*gPR-LwvV+nbGz?eshVR~Dt(S>sKQ(veNWC*`K?D$!BUED4iQxZ!QxRW3x zks`mu&qu^O01}tXK>Urh=Y@lVud>7VV2U3-gfbP-Xnvog+vTP8?oPZkfY%+H$7!c zQxw6H&W*$U!$10DFhtrZys&Amxz~~cTrkZh+OIj_6U&Xo$gMw+gc6xi>daRJ|5*d2 z3c*~=H)7UicBM=5x9?`~2d}MV++u%w*Mf!JC^u@E1`knx`GX^#3e6vs+Sla4Pd|Ih zneNL7#*$pb#S~-b;}Vpm#KFxd)w>ulCt(oLo?h_en~5?c!Lyr!`;q|<6+&P&ljB&y zCXXw4Df~7v5i5uieT%SQ=np?)7O!k0k&xIX8YCjc|L(+#SSc4y0%lBl8(}mu4Gxu> zM-Tb~R|j%6k-C}9K#?K>m9)C)B8B-S0iEqaYCzCY`nuj|jAO-0j-F0(yN!zJPCsSy zeS$N3IXUl>FC;0QRn?MZ`qlGfIu)A|1k&({YNsW( z%!&WJRz5x`CTl$ubFDrfef$;9%sLvYZu6|dOQ!Gn((oLCG8snb0vSe`>|W!8GV(OJ zFZ4*!rt+`5$tu&I^ZN?WOF&kHR*H?^t18BZFuw6)%}|Met{SDHajJ%ZMNc8)C?dte zLYkqfodXN4qj1bTv2Q;&Bz?FLr=&sNE;b9xdD;(WnSN5M(Zv)rAWKe(o$;er7Dhh8 zvZ@~rAn}l~4=x~ zM8)Kwh=Jm!C^IE3O^451N?4FZwk%LIV}At-*@2L%y0MUxf-h-DQpNLT@HQ(5l&z}7 zZPP!mOtnqV#8YNm>+=WPt5}mQTYTirkR2LR?Agr>)$!O52~}zOBYw+{Rmo0IUP9Wo z54W1UBY+BC%r{Nwi-$FYpmdeuzJ6ufF`2-jFmaS|pW*SBQ{-SS4i677FyWB5x<*6X zgq{aGbw64$p2y|ctX#3Sl^&#V z!RbGDVkR#sy&@PJj+(UxQge88VmYZ^QM8JP-3**a!eKPS{Iv^8LUp0>vgG2Mt*(kt-1%Cc)L$BRHEyM zzmBdkq)&?!Kf`OFS`UEav`(vf5`#0E8kxF$@2xy$d6*)t`KL~bFbV%oL zW=MrF-4$$Z&y(P0VTxC+ImX2wRi;C@v2hDfE0p_U1y?~sS&L5itx?Qa2Q;9V)z+Y; zu8uJnib87&)^N}Ef4U)$fr3=PF(M`y#K&UNGvSuaPU$S1^vU>APdc?4-o{*}Mj()> zN<4v*6wEOnn=pbyc{(f&M^_I2Rw!zl57eF@YT<24OYt(B3q6jD=X@hL6%n?uTORJ) z{(F(Pknsgdr}M5li$Tuuqa$1qzw@6YT~9a13%omsd)Z?JB;MXWHHHng8*3{nBF_eZ z>FyTu>&mev*Y8ElS$6rm=el|4M>MqijKjm+fQS7Yc5}O>@`M`a1SHOG6_oCZA>*j(p_exP}M8F)n##2^7B zes`F^A+isAx?JU?r;itUB;9FSTQkr|(h&as(3fD!+hq4g?Ne8#6O~H2<4pe}qYjhr zS(uLQB);)%YexsMfX~gdQ%~pLoaf`kN+%apUA|_=g#zP020~HSei@=%`B6rk*mNl9 zHW#K*CKNmAH6Os<;lb1J@sVL)_|FL&Na*Rs6cpu?GRubOw5uoN!i6>7X;k4k5%z4T z6N2y!jA`BSsk{)!6|HYkS)CxE}O za&^93V>q6$<;&k@+l2~aSN3l+L)9|B{t_B{uHza9SW10iam?m*0+OiBZ*6X?&7ZB6 zzMKSDH`*?2Z?KyKKD1N)@aQs5H%&D73w6(X^ z+Y<=;EdzN#Hn*LlRJP&z@R~2$!~!f5j$YyftXlmmR4G6G{Vj?7{yDN5 zQM25DzBP~7D_K0_`Ov92>TRKN&vBjbU32Oa3oI-aH1a8a-y5h)*Ua5SNW>`VqRamy3GYjEsz)v16^D zveo1e`uY|cZ>@sY$4JZJ!W;?8XD0q$oihgVV^9JbGcGh}J)o>iZsvPIl9Y!ech%A1 zPf-bZutFCGf1p{c(zVDqskxRJvp}2uSOlG|TvM5rDMlSnVVV(A2A0>U4ichDD&A>I z{VYqmgO*vfQ1c!|M-#qxx@PgBRpXEH@VbTZlz*CI85XhwMDAmBVv`bY0vl&k zfHGlv>>^ z8E04=(&Yd1HUUE$6N;J4ZgD~jOKklZ$j-;yAyTUC>-I+y2R9WD{)JSbw`1S8I zuhzseJ`l=(9~^wRwOevO-Dr@m&8oozB*C$R5m_c#yc_vtG>h?e4X-+BqNRKRVv zU8fN&D0wTB?rvO?^lJ4zS#NoEc6UGD;BECb5K?tfDhTPG7DHXF)@$n2NOuelUh;Zn zq4Re9mRoDwQbT-XVv1lNC-Q~Cd!R%~I?k#5TB=g}ZMEJlJuj~^V2oKCMfFpQqoDus z&8?8{C&en2gv`Wxi_4JV*zCpfPiVBTPcl~h-@K#aKDJu_I@|Dna^VOOr6lr+%(a{OWPlZk~zBAu+Pkt~Fu{47k>IU{hN}E@4_c zJlh8mV@t(~_2r+P<>1%}y6qcG6@O#-zM2jL$`Q$J_nwF&FeN9+!Lf4k0&0_~>jRB8o2)2|NDiR!(>9NW%S*tWMu*(Xy6Fzdj6BP)KmeX<6{~ zzFaB0>8kS3sZ;vnXeQ&w!@oKRXe2lSqb}EtNT0t8c#QtRKvLcRWuk?A0KY)*Tj`o9 zH#v*^Ofa>xlOgRr$>}6%TSDhe1q_{i^j0u)%|y_f|I6#kB=v+Zj_S(!_ZHThqXmui zYIFA2J(wC64chlZS93XC+2t>z!?iYOUQyjDm!%y4iL_d#1RI4k_|Nq&pYZ(e5s?c8+kot! zFEn0(DvkjW7AGPJ2p@++MK^*;HRal9m6-kKP0hHJr7`{@?dZfTTR5=jOOa!DcD_^z zx}6*y^~DUznf;`W=w`R-$L@5sE6SG6ib^Kpl)}CR?QCz`qT6z_7!AXz{i$1XadG3E z)$?cRq@uw6VQBf<>Xnq}Qr|4i9S8RZjA)b-cAB5c2Rb%oFw5Y@!wo~ORy6@7rksR` zUQ(!TGIV%V(hbkrJ689_#JMewnuH<^F18j`Te8Z3!BQVQ(dR@&$~aQfqY!UT|I-xP z{i`)(K9HGdvLZo-r|dk{-3}V1VYyz6LmPWV_4{#NAHj+<3o@O5XGcfAj#SekjXmU; zszQx}RtcDnNrPd@!R<>V5fOacj zj8nO^^x2kt>61_!c39X<-Jw-B%V-RI@9QOt(fP^5s@Q($L(a_1&p%XgoTq#?mq+%d zU8OlxFrjLgx;7T%c0LIPOHy*vP*Z>URx1`+;QX<+vN*`ZF}o1$I*7;j+1Wf(O+d6x ziQnz+^bod ztOl|zIcrD|+6Dg_FA1Wfl^oSPJJm2vE=7-O7GG2 zd`B!wW!P?JSsazhg5T}mzwk4a5L-D{xw+z_C))i0`^8|}`yLOE}$|Y8s^w8XXnZ#Gu)lSEq`qpIr zaHm{|)duTdeTqmp%W%^I4nl$eG{v#;=@1J znePtOY}Xns{~mE3Frgv_xVO=ZRqFh6>NhjsQ!puZqn_1x<2MV$py>`abk{qqm!bgD z2K<#xP8wgW1nkzr(BDugS&Ya?=Ua@sR6uSWRVgEL50-wc5kxXJ(^Gt?qU4`twAO`5}{N&;(hJ6dQ8(p74|ibfx*-U?V6|R)%x>V zn;M*nYp%T-c9oP0l9H_`3yJJhtv*KYF&F8X;RnLv-gr0|X5=%eBhw7&4!@_ZxczuU zv2bwABvi`d_CiojiwcRVIdf*hNeuhc-+<^OSY15t8#=j-9`_F3vOJRsL{)`&?w|0O z*40f0h3u}8l|?cC>|4{F(_hp^*`1&9YQ_K5b^fm6@gh(Exv*kTDA%x8`LCo#h0;di zjhnl} z%|)~n_OPb5H!?hI7DtMW-4yu#_H_H#^Y3CT$7NBwA>g22dmT)DukE}-+%Vj#(=qnPZl`bsSS9FrOYEYBcf({8}$a|JwhNfv|F zi~O%7zndKho2vltB8TfU#;Ohq+zPx~c1%QNzB~ zymuX$SQHQWKgy{?Cb5o=&1HXY7xa(`PsG)Rhs+4CXjMPB#h&NcQI|3CeEZwQ!4e)Z zS$$zhm2L^mx&ou>aFCo6_!`q;QDn40`VwJtw8z*pEFN;&AqJr}ZD1 zzuye&8Pd!-!UQpRQPSu)W9b_>d*N2Vo za=M=RYo30+NVO@qppe3PxPAH4!Mn8dUeWYdeJZEL=KB=>-kMd~@>e&337x8|yGtHp z=#805n*wuQvF;a#_2yi@>)}6Z&NguVfCq~-^huR~jBp7;y}?SU>};a5g`Fo`#IMS1S5|GN$Qk)aZ#7Di}lSZyEgeR z{SP;mFM!7{Bt$99KK3A|il%Z80CnNv;6Yj%f?q?CxHJY8p_3TN7(Y0eTe+tR;zx>u zf+}7XF_M!61qEXT#CKi}hb9$mCnrahWfdo$sQgwNOPD;Nv9&qEuwL;xM`?XXj=!_+DRM+Jaa)=MKk*=jMn<&MbE8 zZo?YgcerQ88r|I!u63k0U`iGR1zw;wtjYWBjL4S{0b=q`z4?bH6)1QMz`~aF*(pIP zw%GKHrW#?4yC}VT8AVIZf2V@W2vrrQfEzb;F;;(LRrTo~{Df^?I zPK`BGUHrC~7R$1~5<(*7O{=d5tX2zmcYDj^##)R!n4PuWoXVy9qZkOfD&?{m67aZ_ z33qo1R-?DNjT8s|iHTO(si3$Y#vI7)hVxGA2r-3}0%++`mnBqTuR=x-4grOtCDN~6w8DXHg3 zg9!0gK3|TKe5n!d_?E*w&NqaN``6%yQecIXWX1BZ&`tKNccEyisZ*~t34ZH@LfSbI zPDfSi&KM&RqAeQ`Yi$M(cu7&GC&9Y2nBu)HFWP4ZbDX2L*J(eu>2_}w)pg1<>nv-8 zUjFOvVN!H(xsrXl?{Cjee;2Fsd7E#90ZzztAe^0hGK)K!SrkJMICko~#fka}fUf=D zsdiw)x~2mwFc5a`C6O9XrHH7`?P|N=a0`Nlq{AI#Ij&OU(~=B!E!}6jS&X}h7Mb#k zi!0RL{)_TAci~$(Sq1oEp3W%EBmzd_NkzUKYXTXxbSy!bPja>J9cpB!^Hx8w00_7r zLU4*{o*&WO+`QY`kc0)Yz zi7V&HDqtS}k0Kxf2+K$!~#Ofo!i7$L-j zU?ND1YqV;bEX9?v29W$1fG_|8#}I^0!G!Tr#_~8yVw5PFGWlP@yNnIVo<@)HVvl)6 zuMSGmLwAOPi6asEXgQ8mmFLn0PDf~>>;|wcB1C{f2qpzBTKM4Y|IC&lP-ZC~1zej1 ze<0#J=`e3AQHi|e?l_^jZ`@?bNO^eHqGwkbxW|0+H8(S3Q=IcS^Ny-ty*ZlW|6NIp zM3dyPy^g-hoBF*A&5x8t?*|wKGzFQCwvMJ6gdlwdzyc-(+Sf$Tg{3j@n|8{n*ziEG z#><0V);+%j1ZUf{(I7}aOOmqf$LC)n@2Xip-sUK9nRG>UqjyK~p`*UQPIaa%ftabA zK{4&Q(;rLs}>L@))3I#cZ_Y%?*4T1JPik>9D(q00{5+TvwFE0MsItz|bW5 zxcC;1keE_MN+X>HGQAy$VAEyoekdSOH{k8L93~v+OBBabHA=nQ8FBljm>QWWG}7ua zGqfGajaa4U{#kMgWV=HR6aYjNy@If~8y>3;o@*{2D(yRN&}V`NWGSU%MANoUPK55) zLYD{~`lYPOkyK6ld0cuEy6=sr|>aOvc40G!@5NG_26p^DqU z!`Yoi{0Los+5LPrVD_y+@gLXe=C0SW2Pc6;&AaZ7IZF9!OMJI}gv4CZIX*^!IiU|O zq_*|B7cN8xUWJkgmh0R5(_hx`&`XMN7(k^Evzb8P^~>=Uf!iDFDU+IzpvT!hCtApa zkP7*5agaJPmESDan;#e9dt!@vv3(-|WwHC-*=vgr#jmAc z?f^hQ`h!&dkCH)@*)PZA#(&UlGXCmq0#o7nKR)1XEx$nhjK&BdG&B_3yNpuW8R2?8 z@a>xPi#P(SGJF`{l~iUV<@46S*M`}z0h$6cVY9HZao(6NO64b;nw`R8#qe(W@+Duq z94|KeCa3%~J(i20bn8?9LcrG3j&Oab&07pbT}hT8Fu!Ac zS~Kl*TN{LuO3axMcKI~gS)x2`>+4mA#V(66NO#^b>7Jz+z6lk$vl%Jp4FrjG=@nT9 z425Y)UfAWoP(ci8a37ukCxT^M)K5F%Dw?%BD1YXh+@J#LapGk1L-XReWc&ym_PD5P ze(KO1=3rhiYayhUY&F~_i6BXXJFu|;1|ER%DT%_dWMc?GaIyn1j?iGhPEJ4`5|Am= zs(CS&=_CM2wJ_i#YTZ7m3G1s8gjOW?F$8-bGO$v5}nP|eU=874Gwx$%YH2gwOGJgZB_)uqI#Fl&##=DM^mJPAqxkMD* zcE*uS=pr(W80TP&E$S^XqL!5e;=i3K7mhYGU2=hk-0aIe&KJ0es7eIM!p(%T#SuAd zTvFv+`qB==w9*cu6eNjADYcyk#N~05P*jDCq9`3-RusH zm-m(~N+W{wqce(FzrtT+TeFD1s=qyjcOy{@z9eZ0a^kvuz=$6s(wXfFzY~5w;jGeO z%j`oBdL?lhIVpFn7`#=eS|4*V)hAJPyx;{ zHJk%8bly-LBsH6NAgfM zC8fabjOXAFBtcM_BdOw5R16#%G*7xT+LrTq1VLws{HUK&|LxlW?Wp`NSy}uWZB}C5 zOx-a)iac6Wepi1pHVFn1t0I}_Bf>K1!FWd)($Tofhr`$3iYhZFgwv<_Ug;{9U6m7` z)_9Z+C>Eo>!Q;o_RDjg{#dt1kjo6wABhJhoy4u*6Vu0;Q%a^NEp!6AR5$a1{|GXlY65kS2ycw%1rXnHtP z{p_E_E~YL~JTb4&55vz7fR0R2O^G}Sah5CERb_1AlyiY(~{sTvR+P-}A?(;`TNk z{$bA1Pd(pCFKsPF12g4n`I~i3#;2h$ytq5dB6Xk%de(^z zNrn#WOO;w$|$^ zxvnA%nNJE0N@k-|l+_;nF*=U1XJ=2c{^TX5oiGNbE^{(cZ#l`y0(`^KGj+2+c1zJ0 zST8y$&fBf5#CiCCXJAb4AWPCv7GUSBwsz9kvcn+aCr-!#%v;6rgI!3&Sj4oluzuDH z@w|=$22CM@5-F=>nT$k#NYE+##j`d3gtsJD01_R_r>FC% zHDW4-Ho>)Z^PM>63|{NQi$t+tv6ZFSn9Q{tk{l%5NvUN5j+>GbqC%REX=8@CrV^#D zZtL@>TSBX2Wyq}ZXc-{M$>yzHZLnKnE|l&#sQCVqw&!Vn{il%d*6!L0=LghoP^{x( z>ghlweGN)|!?3)~@+KtBO`rqAYLCN8v;WA{y7U$2PAL`buNOD{^KR(n-;Ics+?DC1 z{6s?A`!h9XH7vt*1v~I_bAj2jRZ6sCofHW*U}K2m8uk z`_G?o&CGZ7xt-3m`udr+j`u-Q!7mjDcItM^Ko^Cp{adOUFtzyd3m-ob+v?7}0teXa$nf?z{4Hn{=rMbJtp3u6I;~ zPtS&tOyC)Ha9bL0xMG4S?G_PGkjB^*!QXb*n#>tVlt}#!XY%rFhmFZkEf393ESEhu zRmJ^gvAt^E?6$CbvQ))^0@4Hlk_w{uOYmw2JAY%Im4s}fkcjIJ>dwRE4tzufl_J~n zi-v_9G84QKg@!tO()LD|Z=~e-U?dhzI#$2S&Wk>=mgePqS4!p7@mg_cZ7uAopziuD z#^CpbdjG$FYwYaZ*;QC@(`^?*e*EC*@ce5dh$|HPi&M<@cad64jkZP6M+;N{^w<#U zX!F)(zfG)9>ft4}Uu~xj=fufw_r6}dCFI3@k1(JIEfnU~I^8=Ox`7&R_=zyZC$f3% zteCak-VyKb?v5ibEb?%wv2o{EuYS*0eyaR*fQl15>#8PUV1O(Ft}*TmjDDPvg&WL%T|W_U`0@ zdR0&!eI|Zx6S>CU`-=pVHh%Xr zcN=MB+?*f3Z^RAnC{3!&YlhXHJa zfdDpajN6gHV-Z5JebdH)D)4yIi3THWDcBn|;z1Q@;v@lp zJxHnp(g!SDU=A1xD@vh+*A<2Ej_X*w(X=L)*f3vC3B{59%2t+mlOzpf5mi#60s4y+ zG)J(ItDxR}ws$?N>!EtF2v0_8;w%+7{b^S_hV?Xi-S}==us@N+P{O zG6b%}L~G&}rHQ!^K%((_()va*!jlrfD`x9H@d8;0P@ zY&&)Fnm@b6?<~K@T?P_M`eI0OaWR>~NGnsGktB^Ik)VGg}OJ4SE$580Hr{YG2)L#myC#JqBG$2-<3V3|~D8)SI z>d=I2ws#y=3cq$-vs&kJ+$hp;Mf8E-h{&nGhjH;@U-8{dd!e^i0muiZjl!uXkh_pw zXq6?UC3l^Y&`kprss24g1E>lZcblc*_0V-pBvj`0we>7+ho@-?w1rw@G}4OU^J&v? zVwbtmn;3vQ3yqnhedB)h_b;#cvg)x=0c&MV*p3$rx^=!kTq(5v8R6{L)&9xM^OZ2D z5E>x+Ywi4qjr!e~7Gx_syZlFsS&Q%n5)-%ZABwm)p|El;YBr@NJ~i2VX4ZavzPDR! zl#{)+9X~4TyR){m61p0tL?Pg*Ai=V^>}IG|+yJAVu4-MlSL5R!?L?!|oJNNAE~Mwj z=BGm%;{e>w6)pqs{K1H%8PJ=^mPGibHm-E1w;Ok(1k?WtEv&BoOkMe7=7VCi6cZC| z)5xINoBR#C?LzD8Zg94?r)Sz#IA?o1MIjmX9)C^fI*PS>ekeaKYl&MHrZQI&sArCE zXz@v}6SBQ`L^rd|6Lesvl@HazK=Ao-&&<$}lanLCzj}}XSF2*+5&Eh{jh~}M-f`vU zw!b-K;(^R`)c>RDE5oW>y1(fL>F(}s5b5sjZUHGNk?!u26p(HZknRu=5G19fOFE?9 z+2?uxAI^0?aJd=oJ@?F7zgk5NH#R|@mj#yC)Vw#s3d|6Y6dDd-HXT`6gt>`(1)mfP z`I$5^8G-5sdkwW##qvMZ&ZyMJ_E;)#sh)w0qIZ8paa>I8xkT5GfG|>`~+Fwuf^SmIQwa|A5ZpC@JWF$3_E9 z{fAo@IEiD+;W(npK^(Ox6Xgbgbm7xL&NZYj{}~3_yO-)?eZhnH`0-=UzeARW*ktei zbASzdK60htqrli8Kp$JK`5o1)2Rzh(xCD-_tCYo|wCrS$(}`O~hWIzN%%8;HS0bDLL_z4@-%SJ zBN9-7T)DA6z~CiB`2zeU>#hUnTO7k~vT}x+*hp>Z(nXqI!1a@+0lt^ZePDXN-%bS% zC-5G;iNGLZX4RLC)XZ@v1#AIVLV5`<+_(@-ECI(bqxRBJR8N+>*y8MKGMLZA^TAXS zkXV=LQy{ZwI|nMxm1YMTFRrP7?1EN3;YfLIbEQ8*g2=)CILnDQiMt>GKlF2@Jw}79 zb%0Tc0zNnd@~`{Z5dyg^3A8wsv5nTimSFu$!}^2+haw?f{+5MeCL ziWCGn3N#{zhldkL1XLe~Mp-tuk)ovcw>g45*AFmz*&{xrxe|q%#H|)@dk%1ru_J-d zfe`g%L5ad#>L3#|L|WZiW2Rfi^q*K6E=v%{CR@1zxkTXvRpAokVcNJHm-Nxag9ola z!@7eCb0Y`Ukp&7OoRKB>#FQsQDZ>pPq^O3tKk?Dc@|cXe;47Os)xZwq8bYbT)dli} z==xl=5>F}2(L`}WL*h(Em0ZE(4{7vj`Kz@iBIg9D6m-#8VAJf70Un0|2IA@rg7X5n zW1KG#2uMmvy%ycRH5D5%^wW`s+m6(bhNA1B+=xP=eCfJbersxKdKYV9rtXsDIw+CC zFN6l;j-EPz7S1b;$%p}#;=7$J*CvN7vk%ypNOTf|L#yB2E%r_gQ~i2 z?2fEPKL1%mM93K|fgqVfg`^)Jw966k0S*!f0|;u#NZm|oA~@(WIq^7~+-E#YsBwB2 zr3}{1EOTSy+|^=jenfS{6LF-`b7VwfGzZ0O*`WAu0Lfok4WF2p0O*X3jm?1)0PW>8 zHB-bQIByxdz`au}mv{r%T8{Bcb? zpB)mWu@13_D}0!TzHFLnyXk zF#WF7`DtZdheIZtSNS^~C_IrNM|r| z#4vXAcU=u{J6sD9jZj=P+3rXF)8D|C4coq4+2o_fIqZp_o% zn_?9091u$M`4I%UysF(A_n`*a8z1+lsc?E0Kk+*&Dkubh+fbS-NfQNVfcVl88mhy4wz4BRpvz}hLRsy?s3JTCwnr-cMmS|Vj3 zCMI>7fCQ$igie&B8?XnyKb(`8`f#@yNtMoHH$~aYXFt91<&#ULM)7d9K~oO+qUMlR z-k+`#3D}-xAba6920ZzNeFfe$_Z2C?PXh))MmB{?;s(6jP?ZaJcRonS8AF}ncTrJM z;Ad2ysU&jBWh+qWi!ILZCD%UQ{$428puwhDqr#93;Iy~4rkxBp>qDFw>@U_&wG{bT zqJVQTYc7~&J-G1^H~9+O6mmIEzpAqpz5|8U#spH7<6O3!*}niH=tXKqM)_n?jdEG6 z8cNCja=Lj{5)`$sPf2I;%AyY*oRd3`de2k{@DM%1JUdPE;8Hgrj4>0Rfo=+TL+A#= zF^wc8VbTm*9Ch3;@wla+t5EcIiZ->GRcOCwWo6Zxbfjd;J%)3Dgrj&~1^kPPB2Ch9 z%%FUSLvZ#=Y514QreKN3loA1Pfz)>t3p0ntM+jfe4w@%y@IOwte^O$cIIi_whfSjt?L8!9H4 zig$71bEWYB8NsQBXL;=(dgc#}v^EZ(L~*NVt%gBPm? zXgZF6oHqkils)~DgH}mtUwH0_mKWltyt0#emHRT;CH`EAI<6X!SE0AN^J0>aYDR>I zQ@wS^OyXo6rKej7)}JgC{CM0jQvXW=I@WkO+=j z*~DC~O`QjQ=q}YNj-Gj<(1W5XgP(|0Yt z!czQQX-zz~*X5E8r(BYzrFoPij>5#z;ukDZ@M1Cg>J)u93N0O{u_VpQH8FE51K>|B zDoXZWxsGk6P4F~9suyb?{)Ts%W5}ak(>o#l#FSl|MS9g2n*HR ztV}>qN6l3f8q^{%bHMhTnxgX`>qDl#R~MO2PH`_3&yhpEidv3d5HJOc;sJpe`xC_+ zAA91Uw8L=utcKAD;oUG%cwy16ti*!HKfNL1s>h|kR*yx&q|g6@$;c)|A6yvi%Rs82 zG|?>aB5zi$(3$LY*d(uy>8c7B_q>>kLw3-TN*OdUI0k3^MKu50FoALa99pB&v-Lm1 zczC9l1+&1NAj*p!5MeWR|7M4SKCml%Xg_wojEU8>Ko+ct5(=cfKmd|$>L6h(VOAT~ z`dA56VGwY|>bA&X{YjlHiE+nP{(VSLDrEKl)le=-8;9`kK?x0<;j5rb?ZYn|b@d`a zt5|Hu0=aRJreKX6$-L^~psc_>9~xC{ih#L;@B^>(AufQ%{x8!{5f3_H#pBol$TM@3 z_v$I-j^kH`(j28qxg~lXz_I9=Qnutx_P}36I;Qgv3K%&r}+I&j6 z)Bi@_j}B=OHW(-&nL#~US0MIL*zPbK{)7udvy4(53Q|}@F=Eppfpn0?p+bxyH%3)E z!FsUrA=J7wdL&#dXpMM$Eb%dRq%0{YJ6>=B-2*#!I~hUiXJfeX)sC$1G(R9^wxi7YV7-wfjN71Bxy5nl8Dqb3Yn3Y)IhRf6*qXYwldfOCA-g!z1C%Whp5?^4|n*_WM4E zKDob}=g}+WT43(9ckKZhvo&gixQ4 zm6eyeQ`USQDym7)=(yL$EfE{6fsyf+QRs|O;U^q4DtK0gQ01$l5icnzDW6|a#3n=w zrF#;m+YOCB&%=gRBI-$1hCcd#^lw98{F};fyYp1^aJl#50rtGhY8#iR=vHIyXx@H% zfzkm12em}Ls#t1PS{i0SR&H(_e+x+MOUBxKe>8`Iiq5`n33|yx@=~Cui8WDfzb}=S z%f$pcFXKgGq%c3h5Pn1E8@}RZo_>-u(@RW#qIdo+4W*{#LDBVM+et z3w8>A*QHBst&1_u_NGQrNNio*E58{~K$Mi06^LfL?yJvD<#n{WtQhcccltJc0ccje z%go#656GZ^lEF0Y_P+~$49d;=De71d+9dMuMrBF^j1t}YgxYW*tRZIDj+xi}Hnp#py$3sF(^IUBV~+r%Il zN}@A{TQN2=xQ% z*$|keN0v#iUN5oJ(=|M1IPHqr61U<3xrEcj_M^Y;mv6(uTiz{vQ9a2iUCOeoK z@0%?8;an{qnFpcJ`(3j@fpW7Rt;nwu62USj9u7Mr?6&=3$ak^6JL!6Z>|dJIGyNc% zd4N=<1+gDECXl}V>OQ{=0`-7cvDtAcq)S+hqb_AO*l?Ywlsg>b?Ly3I({1lAn^B;M zd$UyJSGG?A;q)~ebyx0;fn2M_pp2v$>;pw7XjV%jyXR<;(ojHmRtn>xpc_Kj=JI=3 z#i-RI<=y@UG!Ss-!mzf-_kQ+-7v9d5t;(2mT&Qff__@2oqHAHc+$1Lu$;YDeY^htV zUvz8<^u_y<8kk9t%%zPWq|v%%XvCoy`k@g9R>QB<&({8*blD7BT{icQ_D)SJKLqPG zUn^=#_ZAjbc75=Q#7MR*dlIYqa62{miQkLyVt0{lkJ|L(wN5Uq|NTy;C4q;{)wUT# zR|dYt2*me4@;@qNu-~3vlMA;6PL4)#Z~uUw8fe6o$_E&%Zk<7o`6e(&Auo9UJ*GrF z1C1m|P62JU&0Q;%3ISX-q?IIYCZ%`Q)%|tYLh|d~>flTi`R{L*1cvxOXs363dwWU# z-Jl@XVvy12iScbjh~-YVa!e>?vg_P3Mw`TaM3i(SfKx<=meVUvC&siXr{;&><1qea zF)*BPvW(GI^lrWq+9nwuZzJpS&gVm`%e>{;#xv8H-E-Et-s(Bd4KXUf&Va}r5}Q}^ znV<_aCz}VFVqTVY-AY~mD+lz4W#91eUDqvL-cSfmmwEq4AY1--=abohfjejHg?{v_ zmd}5Y64+#jcwQ1>#EjJ|Q{o5M5l|r2rw|#}|9Z`+_x5Ad?4_tM7@+{H0KI;2L`i-R z=h5H5gaT0^Yx5Ud6_U&%6^Yn9sq&%giFztnaK26D^ZeZ?VvIG2VS$E#YDD}_i+!Pp z;3Qw2H zi58m0Pfss-?Eays#T`cI$QmL4fRcS_WA%`t9yJn&o4+l>`$90K;>lasS=Hp7$yBFd2jzIkD0)W>W z2RVoN@%dh$QN^e=efleO`DaUU&?rZ}C(pEWo_F6Yco~{=ckUD}-N={DRdDq&~R-fVyl3^jwun6&=y7pBMi=Mar z$~T!N+dI`i1L^x9Sxaju22Uv-HGPk#gS4#5v}-wD3W3eIZ9m)XPs~)f5A6v1ZZt#w zhNSPRom3=1j;G)C@T!7(6K=3c*@4RW$6;&ufNo@$5;CXk0 zo_Hqt`QkeV2r_EzcKS)0ot>RKDOix>mk`LFtJdSg#lrK+1D-seJTh@O0;vAuMNa=5 zjgVcn@vvo7ov0Y8PkRgcJlr@p**=nE8(Qtv{&rqzW@k(<5LJ*k18D}3Ww#rsaV!{Y&{IU1-x18NKl5?#IQy}NZ0J1+3+legDD=$uj6J=+E zzCB`7mGq78;ne{u!7O(Z_iNE4(bvCMO@5o0aJXeY(PhRG9f0oU`h0BXO&l}Jzk9tZ za$ItJVtjl<5Qvk{n*~Cc0ZfFo;nIHnbx_MLh+42M2N@T?$CBU@6ttBe(%K9%XRYO? zsqIsMN|+gPCwR8C>p9HK+6+XZxy+S!YpUk(;<69!haLrQNQxG@qD|3CSeT9ZaEonj z^p%7AhPFWH&*rH{yW-B_DxwA-WuZIWyLSaBYo__+u0&F;EcB zOH1<<=DUW;S>D(gY=>(8M^UtQFc6}RwLhvSK*Wo7T~AAENmB{f2J zQwnq_4kyQ_Z0*hg?5Tfs{Dv^yiz=^*bGgNNQU_ZXFa3>M06s0ciGHW`{nYCZr`;d5 zTO1{TFE|V2O=Nz=kZ$r`02@NyZ1^$ zo_?nLcbCwA*VQ#hM})bZxT7JJKeAglc9RDV7ivEBoPj_eH%1t=c^c8ot-|yV zv=?P*gkT|qw~?&pf;(b}zl1qD(IUX#|NA#y;rm+g|4T)Xuy#iVUAU@#Rfmh7cD?mfS1)vq4+``8 zyL>&QxBjczq%5Gz@AWrtWTbLS*s^8a4lzAhS^5zke75JzlTiuShsbb*qk^2UmQ?JV z_;Ll9C%tOhFF)>%R@*PV*Mq+wSIIFYiD99%{~@r*dDU2FB3%2fG7N0oLS;(IN)0(8 z`ePOTcS|$P3dN5jd+`D@^YbT8j)v0KY+siUu8vIj>MOfOp<*hH%J_BJFlXZoni7YG z@qeLOOQ=e@n@%I4Y9hgV*)Q;Oah*@MWZqg5jDMk=+Vl7KFO>T5zO&v%u@hX(ARY-^ zecZesZcf$Il-~WUEL~HbT^_D6X${!PX#tVL`%7Ish7wh3GWtdZOIMuu_~3ZJ*k+D4 z2zI1 z+r3ts3@cjHtub=_OmXXKn=%G@kuc%nAgQ2mrPyj7SNg00-M@41joE~$xlbqxQ?<3l~FLRxJ+*rO#YOj45BYx5&G|6-Sw@wsk&_amoEApjiopH@@0Y^yV9XfQIu!xH3PI?=A9L+1P|)qn`D^H_>#US(gjP>X>}SzL7*QcZ z4mM0iaESE7h%@b!XVHwbMRsWnz!;YjI*biZ8d)U{fe3w`LiaJ#80!))+gbDBrMqtk zY=*o~-b0nUY=-(aWLoA8x$Aq(<)CgHOTLFWPnHLt$r%-7JnRaGJhHc8IA`dbLHF2g!)+YrIYm1fG*+d!#UFyzt@>h_zmc*tij0|+RaWl_*n7) zPy#|7$uKYS=d1xa7M-hCD9wf(q#GR)YrJ>ohcdo~G05aFt4k&gM4%qpmHV zN&-oXVKSDR8$KLJ?H9t2s+2l|6FtC20ou*fCXi@*Cbi;IhG2`4LCsuTGi z>nl#{lg#~Nt#-9MVX6#xwww-^evUEm^GS2#k-&5&0FsR9>sARRaP&c7ESdUCogtaS z)iZJf8rF3jRIEh>(8`7$;em$LnW{ocY!&nwZwtR`%7;GJ9c;8e>~lpP>)y0MLbS&4-5WIS|@wN<%V;K0qAwV-L|;G`9_ zen>Ep5q(sjS?_Ta$2)Jay%c9)VDPB&0P`}s!vHWrCg>XUDMN?WPRCcnpf!1RP>q}k zk_$(b1?TaS*^LPpG#hwmAu)>|$-qDg@F1NQ#h{jWK_5mGnx6DIY##<;J+EvtJG$q zqk{m?0D}Gl%xrd3xnnIqj6H!#`;F6LJw&+g4(<~WbJ?fH%IxC*lqSR0g_TCD(hj60 z)A>@lNlAk(4Qp#_x_lu-1;vrcSXk*SI?c?*#dNi1ty7JGdV_?l~X6`*Ui#GtTX z--V%Nf=;3m+Lukx1=mE%uJsCqN7o@tce;Eb&g4F;Fy&4ovU_l0*KIedb z8#Wn{OCa?HHVJTBj%>lcI{@x-#Tvtw!U^F1YiG|NHsMLU$_y{T!%s}V!3l&U863Aw8&IJKmKk}4pup7R@ySa^Uv>V4n)h`bY zFUKefzm_)bP@TkK*!oe>qn`GSS@+7@@%$>2|xY6*>IbZ#tXhW?aH(f`( zl&!4oa`+YVuLp$KcoEZf#ZssvZ~T*A^ZhIdXmt(#Lx);rYWMf|`1#}gIDu~h1zs)8 zX;Vp}if4<;ZWw`N_!QE7^!nIf)RoqEiC8%9RR1*7v)}>(0`Ads1n*%{8`I0v+X0r^ z&R`%V1!IE>h?5h3)5Q%N8#}5={dt%#C2cqiQlSRXnUM{DDkV?jg?%rGQ)Q~4sNBL) zvPgXkUVpg%BK5XO#Mv)QsM-WD-vn`zz9d{>o1#Zx8p4D9aA1!sVXYPi z%Bo%#U<^UvtMuFlMQFadEodE{Cq7}D_Q2hUCtX6sV~h4rb4x)N92s7i(b^%!9{r46>_`9YEjSKa zYxAwm1Dm|i=DEwghP3nB+a{xq*p96rW+1P^fpmhet3xH%F*)i`4p>>Yt>|FX$;x0= zxT#LE7hS1{FJ@94RM$4&SWB>(0@WpvOrhv8PrAmwM`M$Hdt}G^x5;yJsHgd-Vq(9@ zUVBfxK@1wufwD?-s&G0Pv|NicB z!hRY9kWFhl8#`;LQ-)Yg zdEs2DpPyflq^fH2cDqt7yDp5u;Zqt4xSxA9zi|@9;Bf>J$WK3Gkdx0TO45519LI2pyU>lgkxC0lJN9uC89&o4JqueLLuS!igUqT>BsajfICxqLQ>$qQB0O zyM2lWmyhT}o{zX2AhzW@mB+(lcl!F)+g6o@{drxC`}nJtZa|KQC>jTQnkDq5QBb^( zk6vg<$OMNW?JpN-=>FvstNxCYl@>CdfURSevUczHJjTia0evAdfoU0OU%CC7Tn56k z(o^m~d%qlYzulYQR;>J*7Oz>2Xa^mUm)mEZ1|Pn7jKA3I@DupgAp>c?y4$e8Hhm`u z<}(xzs+awl^)JF1QAC8qXe*dB+^UWB(>S#+JiMuCene)Y^?2tajGlmAah!Uo@ZCr? zvzu%4=})36ZN&ioH>maBg+^X?KmR+*J@x@`_!^kW1QUsWkp_?S=~!a1u_LEOMIzYE zj^0%+I{yRYoIO-4mi zs@CvWs%(0rzGZ?SBP*l3z%WCtDM7;a>yJ0hpQ=zFOm%x$ge;*^8=$!SMJ3ZLmIYi1 zv9Uws?B__iP%GpwwWNaT$MTsxq=cK+Jx$5;q>Fc>eSE!QmvHoSOdNZE)> zHE_y1+}&Nq3|MM!_X#U1`Un|D6>*l_XbGKaxKUOR0EtMU7DT(Jc)1%%W71lzm|wiK zQ?9g#wP6=uOiNswoTQ_-<&f*xBqOdxwA*r8>!xAhSy@>*j6_aNNfCMP>3OL6e%)@M z2HHkfH$%#1R6j4nMcaT1A2n+dhE$jz4BLX3{s0bUSLYFsqtc92`5Oi%tYu}S**khS zJuL$t7r(uo7&>f%yQS&Cm6vI3IrwZMyJ;kW;o;7Qq@yb6xne`j&UV+8J-x7SjWy+~ zIO+zBY8GeT{Jak3^lW~@G3(DXH8@II#+>wY=QM2PjPvtrRZ-6{D#T(GL*nzXiR#tP z&cec=2u58)-Hg1vpOZ~SQd#l_4M9Y#8$hJ<=MN2|J%Sbqi)7HWek>n-mz5m*$Xqk8 zRrU=A^3=qX;*WZesRE+-F>bup^3hSwcCGhZ8;c`5u$0u}TE0%l!V?d{Vig2pk-Q3Jiaqrh(pIR7Mnd!4w@AjbyGaz(OP;l_?FWMlN5Nt!&U=zBA zL)AE|A?cSMkAcq_hXL9YpgS_qQXpst*#u!KkMwsB|7`ZQadB~pOA%+%#k!<01{pY! zSIR>&r2Nm+Efw4p?AOK6HKL*W|0s-^{Qgb5I)}-lxJ2LF3n2B|`@I5zA{c67R3uFP z^S8Z*QJ`s~DEY)+#rW(Pl3jf z+F@xUE;ee`AdE8c9yjt^3&(nRI3#rbbO)pk!S@#U`$dD&R!bs6?=en(iZJ`B1>*^a zBO^yPHhK&oM)JAN(bDmqOdNPT2Q)etH`FCg8^yry4=4!Kj!{HFS>U`0;jb}5gh!xf zD8s{h`YQ-8<@xy>jhOeW|K3A{rWmcLm6X^^dgYrlVsR+?I{P)yht$@pva?TRXo<{7 zvr^*M3q$_2CmO4=$tFCO!<0Ck@b~+ho1vd?2fq+g81U)p(#-x{ZRxnv9vj`<%;a;z zHnFgnnXwuHCUR?d3mP#p3p#xJVDO$g{fj)^I^I4wAaDH(rS&Fm7NN{_BPgJDYU=I6 zJ9Z8Zfl4PWEC#wV>C+f;Ve9&;Dy&E~6HO!av*T=!*P%O}tx7p^+$4Q4Lg|?raf^FV zH2m?*mPl?+PRC$I8+s8hpNfl1?e)}5)tDem7@xtg|9xEQ^}%lzAvV6Bqv_Y>qg4KX zh7+a)vm2a8*e@N7%@oQM6?JP2MzZ!d>YtsL3Ej}%y$^OW>hxY0Q>$~CQpF~yaUmab zAVp<0`Q{1Aj5A^Iwn5Z)7XN^FN`SdCS&n6~@&SmXH_lLg894v`mM%4dHZv&A3kf3v z*&)h|zhG_Q5!zfaD`>UJ!GCU2B4H&3Acu84trO zWjjtwPY(uYdCZn5*k240n5AW8@E5to#KZ{lenH0L47Xo;JBO>wms1uR`th$R)x*um zD-!O-#ep$G2NG_3$`7t}hAkQB)vsT(dNwtjBzN5WO)L=hm27p1uSZqZRyHn6AcOt( zK;-l8{5`UgZO5Q_S69)zP3B#;WZgNEtm<_7SR&kH@zI_m0B z>Tc2#R#wiE9=j@yIvYeW+WR>Y?LBi$ArJdp!h^ZNwIP# zzgKhu0@o(+Hz@u8YzMQbs;VyY^zDJtUqV?KEe#d{tqoi?KWAqHp6~6#KEv*Le_jK} zV62OL88szu_I}{Jj$Y9rP-_x$n|1pO-NjcDuY7#Cal3&}`~B^G2WV_6#99SWU;{HX z@RZ#q`4-e&0T}~Vo|K=HGmfPZ`0}tmzZaED5%`$QsoWFr6tu})d{G_~0W`1@(EM8P z^|7(BZeR8$vWvfj7t?3C&Y z=5d22TMHC!EG{6{bKE>Qh;}*to}QPd@*~KeEz)NqPeg#RJRg>B3(L|pw(Z~T(LhsT z)q*V-sVSF*ahO$ELLjS2a$c&^N1rNI$2m^hA8PW`u?0qgSPKIb{KpIB&p99biwp)*- zESP?X-!jBP630aX$r-~&F74rP>WIqVl@y?)gyN(G%r3NHsM;!wy~Iy2lEebo7^GR5 zx*dMfhm?zp5WJa(|JD8Pu&NvyPIKp#*jLOocw&dcy=Z$6u~P@{}pK2IC(X(s0dKC^(k4w z%!{M|1`I?hIr_nkVQ6hA5-3<|Q@0^Jph2T697QDa<93til@XQf#g+@}=p5_a1k>9X zNAE-YzPral%OKozKVB$}Z*ALHTL)1<(zre2>)M4ShzW4?6fP<85Z}G~D@X1d)+1<| zt4l~gfEWaNO6WD~&pdkq&|~|ucxMFAY$zdcy-;_G6-82!DM8{|(S||1<>juA%y;C{X?zc=e{F(NnB` z8=wjiDR&i%VTP+1+~D?r zl=MrgB<{kf6kMd0MZg|)K(}78L0z#h_d#+Pk@#2}OI{p){~zFisUl|A*UjD|o^$u{UrQ_qo~0xD6vD}i zOH5}PDGK{bHTnm*XR@1j$y}rW4 zV2{{J)1@vK6;UbIK!%nqp0F{r^h!AWDPV24?ShfHQ9;5ICt&;Z@BPc|C^fEiIiJQFSkB%QJ_mbv%j+fl6+O&TZk2TTdc zzo^W9kMd9vvF0?r_j6imSWaPR(r|AIi-1UDqhPCALE~GWZXH6fZV3x<`~}cn+ZAG5 zQ8z2CVz+Akh{5Da)7R;#hx@tdUmOwOf+cjOQOl4Bi=&~gP=;^LAg)vwE`R(}?Tv~n z-XEOn$%zST%(SdCkg}-VW6>XbSX}q1Wb$2b%#Dx_rV1T*CAFu^@erYi)P5?x>f`B3 z-Nz??0yfh|5#fl18GW&MNwv2IjZQniPLc&CyUrnzTl)D6IeI<86RBF7uZ|Nbh{Z-{ zBv$(nmK$XVDEJ(4X^i}zh};K;ED#X8Wb5UKWZv@8Y8yW7oStnCoYS0(24Oxo+VIzX zTT4?9u@f}q9MRguXCsy`rmTHMPTr>@qXx4LW!t>ve|vr@E|Du^4(p?vE4S`Eq!p7(6=s@X?y^6_ zU?ls3S+c9@@Y}?JvN}u&WgtO*-?Yn5<1I{oFG)Tr?-gfOD69u%3`Y<#!Qi}=Am$Y{ zFZx{tQM)_?4grzkpoVe9$Q3m+k~nq`{4){8AYv(g*4wpCU%_hS0jivMn>1+^t^u4a zEMgZH4|QRY_e7-l5j1$S#gS!Pk&_`IfTbfS@;9W@ol%d_ll1Vu1`VME zZ{$tjn4tSM7|~!9qok^mFIcN5f7b4S$I}hvP@gmg!_0p{P=obyx%O_cIUUK>-i(Nh zx8Ft5@HY$^(|Lof6SDdy&o~2Np=P-hr|c)RemgyAywSl{^0~pfEv;wpMrq3}sqOFg~+8$GkSC+Sk zdS7-r1xql7My5){euRx%f@~w3l9mF0nbN*&#x*ncjGHD6V3BB_E2eTj5s!aZ<2sX5_TZSkcHw9(`tb7xfAyE%iUe%3}LKYBaXM}ac4jK}i zBk;zL$U>zVC^$v>fm@=-O!RR3nLVf>JbVoopVIH_De>}z?6|he55F#)ih92enuk|o z4Y~iSQr^J8*Z;APZN+=`BdM4Z(xxtKI}vpQJ@MKyS+a)P!%c_x2{D(SU|_oXx5&6; z%q@mCjd!eZF$o4Owgb6Uwr~iZbqdg91gM1e(4u%^tK+C}$amvah157kP$YO{L!r=J zgoZR!jnhC|L;8gBylT?+BMX6-@lBP3`*ERoIQC0WYVJ zA?J(?7MCe%s8_TUPc&U6Nm31Fpg(xrl2=)ge$epC=RtL5oL$Bsr5MDDd6)=&LOD#D zw`kXR0-S8(_Ju_gieCHEoA9yES%Lv*KkW{2(NJJG#|h@t)zd$J&rVP8qH(`XE*|L= zeU0!6C3Zs>#;Q`RA%p#Bw42!@u8^hc6IH-d8Ah~{*o62^hSH;)hL*uO!=s8puZ&#Dgfh9 zL8V{>1r>#)CW{z7-Ji{?A404cV@i}YT{~}bp^&OYr(03BFk8R%xzKcPIrd0J!l1cQ z29r-FBVXJ=K}*Ns^HrJS_S6(|^=;+&oZ7Or{pvqDr<$QG^5`95V90L{`HpT! z7*>6L^PQd&ur1VlM#q`~3MMVKmWq&&n3Ptyg(PPoZ<&*#h24{#u?w^}X7A=VYx-*f z-kifM&C#=;Lgg*>_67w%Nr3_5`fKWWpZXyoMvSbZc{3X|A zGI~%GZWynFqn?1-dC=K zeZ-TkK++^|dQpMXB`PX7qR-wClx`S=M2<>}s!zADYhxL>tp2dZ|U6cDYKeWDnK5disPnMVpkviwYW>_7HWN?ml7XoX|Y)V zPM3&-R(9OJCpcge>_7rmLs5J27cMLsI!42t>~XcD7ARCAxj4eUZOzKwX<`pSqN=8~ zL;aRr0vU&zZ}gLDz7%GEHHstZ*6S;(KCzrDe}tEi;9HE~u%|@PB#n$?VPx!GSSYJ+ z+m|-JM}PlLK}DE5T1-bDwG6CefYFP;!v7_d$SWZ5Zm5zk zpPYOkG8`4X>l0F|7~6J^&tm&k<1&B;a3@Th9fcARkoAn z`LVM)Mds;3pNDl}Zxga6KoH<}wamReg6-WM==bpO;dG@oWonrPw#6v?>20n>CpONh zDG9Zv?T23j*Pw~tA1-Dyz9lG|A`=n*S`{|$7n+(QrzjZZAa=Z6@+C&60)A!z9`(hX zcPFRHj{8)9r}hySRlft?etA%y2+U0-aYOuc<^bS>qH}cY$&J3xL*FkhFKe@TV(C0H zjhQSki*urx>AfqEP4|8@t;}XO*=1wPrO{94#RE$b@8S;D?o9s**oWFmO zqwo#;-3*NNp-Q!BoEk$cq*AFSaU9&HqqLg*7T-W4XYf6avGb?Or&j5cB@uQ@Px&9@ zAtCfUJ)ACnsF+wb136z7if!IfrrK2hxd#QoDUb-B-J(Vj3PiIpRZBSxG;8Z4v3-CZ z85wEHi8)!K_Lh^2BONA+N0TNVmMtHI(RF~1zBlT(O0A2d6Ac&kG?#wrd-z+x8PKow z65wZNJr^R=^CL@aerV?hddLx@6MAc(B!-XGfVdZyaYulbwx`qJ=%z&|!oGJ!eS}|B z!ek%JL^6A&;CjpHwZA%N={L$Z1%^4viEwZ} zUYhHhnmSWQ-x~iu9upfY?CoZmwrXX5ar0g-kf|DS zL-&7a>5-USB01afy%byuOUHIna`1HN8sj2(<;-^7V9r`gAT*xogy5G(q z2K+GnJ2uX%HLsE15g)?T^VvCGwu@gy06DDFoqP*j{BTo-^orMsrPXKMI^%$myjwv* zL2jB_e9NaA{+bnB++C1C8dwHm=k@-Iz1PKb!azF~`*>*m8V5vvU>f#WSz1uYslwtl z!zH?$HrWE*{?4))0b9191)80Rz63m)Iv0b0_s-$_0|uTl<(%HO!n+WJMP+$q%U~3W z@_O^DQ7jW-?@AN#BmSCf>0#Sn5)#jV3N{~pe{!{KvfAkUX<^6OE|KW3dj4by;ty zYCF?D;mz>X3!3IGP|~Sst{L?yGmw_B_dJQ1RLZQgeweh7LNIs`C;l)}ZP-*hAFj(& zree|1)1p^)IFP!oYZr`K0|WcGH2PU}Xhp|`<;O*S(gwLS`%%eDY+*}N6Y7Kd)dAUs zUV_9+(^eGIms6A9q4BtTe@^m%5}f*J*wv@qwxl^3?<$wB4HZ-{5g9RlQ>#y1*gF8ymk|KNsvIq z$4y%jx%mg(Pgf;9OUZUCh`_Wfdrd=Teh~e)KR@>J|G7Rs2VqFiySVC9T2SF9-h<;g4 zqs^7iYBpQ%7Be+X?)WV5Zq1MKd1IpOhh(0@G((&c_8dT6XHL+_ff*j zD@*Y3UVE7L)yKyJk%?SoSK|E=s$!Sd*B7_fM~A!p?%~5Vj{x%ZxV%VB)k+{!6VA>a z`ii@jpYIb4G|6Pbi!lU?ZNKncETJ33tqcHV*}&=$?H?o+u_aF9EzlYaE2)+{N@&yRIYKJbH}xG$O-#p*c@<%wp9HYH(6k z?Vje19QRf1zFNh9t!^?39rUrgyBW`&+AXlU*;sRdEz_gQj1$W6_g9wEEXQN{2JtpT zk!PQ)zrRzIg#`{56LUD2mp3<6Ge!)YAMKe)e{rR0eSJNp#xKBG_d`>lv*>${()zjc zvMMTLOH1WpjF#&|-*5UoIJj6Knc&->Btx!~ z)6!b{CStim9vd4Q85tWH8XA*P50k8WMErJ~6y7L(-16A^DC$MjmD?+;+7CLSB4382 z|BjJqe$6iH@9PVTI*h)qrWWbLsr@lt@-po)TyWoEN+=_Cs1;%o{%7H!%FMyVepaQ} z;kN+}t%U3H`8`;K5BFu(jKj+gF17t`Z*Tt)``6~{+vK1pZ#(S1K7Bg5y1H8YBI@t{ z4s}N}9nZ7Cqo_iapi2tdrSVR4?a#6=)L43b@j@lNo=6=UEop4P17EV?Aq6XmG}JSD z%eNJG{AS)1d9okv5HnL>0uYIWc6di8@zEWls6%Go#+xxt@A8e=PM<=+eSCvIG{rd= zMk!1F@wDNM)r!SZm&dZ0Fzh?+1|Z)AoFN@yRP!9SwPxD`eG1=JRf!yZ+@srFe^-^p zK4Ri&eI2hHems}CaNK#LoaPRNex9fvzqe4LIUxXe2_;KL1daUs%!qG(FaOY}sXUoE zP?y-tJq$#%+<4E+JEe&v_w}_{5Wcl70szG9!OEG$ z7)$N@L(tu)j~n(arJv}86Csba`1NT5=;SgvXSfubQ&2W>V+C}d13V0;dSx2ECGfNk z{BmoYTc2R^lpyC)1x#1I?E4Jv!rvaU_MHSf5DG|qXJi%62eMo=Nd7lq!C@)TVE{qU zdsgT|n+pB!r$oR7IG$CKyp94FdJ5R+(+Z;GJFGNy-~YZPOA@K$hq2U;VEy-f52|O* zV8}O7uABe-8Iy_+}zL)CffLu9sS~9R zHp~`J_Q=KwDI+5zCPx1qV7kM{N;G+;KM$OZKOegvf{j%SRbnkt4SmI&?vI_6I`1|r zM5NFQZxX<6*?Zoco}LCjm@Y(f^lR&Jxdn(neNX22H2~7iuLlq1^Wx{@iv=1a+3lVK zEI+onPV=cQZ*Inbz7#NMv~gZ70t!poVQUOxYQuSQ4}tsKZ@>F0}+Wl)=gl zf$mX)`cKEs=+Lk*`SpDOKKQjUMW06?5DviW_LiW<4&V{L4iYA>Fx(cX@ee1U4o)1m z&&dOqPQdPZ2?Sz7k{b&Pok6?sqKnJ^0caXP@)-aOZB1e9{A+QuqV+^1TQfCsg~VuT z_lU%efh>Wu11i9pfD(>4kpZe@dUxXg0(cw*cCo$hi-kX(gS(&&=)S`3u?N)qBm1nT zGbjHUvu23u+SXP)K&4Ro3knJXbiQ@AO-=y0mwVyS__&p&r6quRHN%T_ja@n#nmSTZ z``5r}s+k7(K0F?;qkH}Oby+rY?{!|)5abP4V4Oqs3gFW?bn7{C+LAK_VJtb+tgNA+ zVe~NuRC9ELxq}Q14OyGxfcAYG6#Z}#7&$h*_}&3{8}Y~61JPA{Ipke%1fSO`7(*F$ zw=f@T2zu}-gl0^{?6w!~#@O9>Ta0 zh}uskdSE3nF4+ud6aaLIcU^<10hkl*1(T^g48$F|06dn1wY6VTD3rl$vGBBLQ7;^; zdlt9cH~{M9u=WfOn|oA(Ge$J_sNAez=m82TzV+5c4DnwZTUTRNNxSP4uSZ8&o`bbR zhB)GLO;iunBd81^K8uhA+OLopZY|+erg2okaC5UPG4O&wd7@EEZm%#SHGjl<{lGb9 zj($a1lrSCJ*Fl6@aEc2@^{kVvZRHmXJJ!;2_KOMb-^oK#*PYdcq7I!&Y|nfE=2Zny zn#p8xu4@d2H9(&Lx1lw@8_5FrLgyFV`dLH2kL#)ix=pJ~+AAuxHT=erlcr}D?UXvo zzUbqW1YNAr73X&S@8-R2;5Kj~I76h)SSiZRPNM1E`&kM+)`txv^K6>xAIXW?1jD7= z*gUAFoD#M>Ma&*hEKv9h>JKc>d9Ch#hLMhrL9zC5fBz1T(q_8G_qmpU+&8|~%t2=j z_lGhBCxn#B$OwUO?DiPtudRD>D^9N0O}L~Et*1e)WaoLi++HIN=bfcL-#ag4WN4gk zZfm##K*Dg-DkoMYyPQXdWswh`^*6o~6IQ(Z-q>6$?bp#sIT(YXI?eFr?@sC8wNpZ$ z55Ok&h2@W<0a%C3*_;XB%(yo$vi9r`P|;-5Tj_EG8a+^5ECb@*+gFvxiBfS!WQxYt zi!H6yKX>>b0{i%3?iZCV)ou!G3DkES;i}O2TJ7v9t;2O8d2xA;9GEq9R0V*|zL9Py`W}GPZJPav5Au*zAK^>glYJ45mtr_5lRiO1waX=|kk~ z#f6xi5f#E0Qg+83ZUwsC5FP&|;J{TQ&EZ?Z`x=3uFIQOzp=soz)}kM0Z%&ewmS(Xx zoj++K^J$qE{K$gC^4?SHa8dREnUZ2Z)#Lq>1)*PCAg{lixi>nE_`%M`CJijDaP+5& z+KVB=;g_OIuX%FADxX~4m`qZWUqFp4Gmv`~ZL6=|0P|4HmU!IB7O}?O<0w>a;nN16 zOXQ`wTc57zTHtkG)SQKEyr1(L)Dbd8cpeuT$|`tbYk^CDU`#>fY%Zg3gjV!()8H^X+egb5aMOD@jZc0cF74oq&dNX zku^J2TlT`ZCKwcE;o~=u_W6D$Ur{$IxGE>>v8EfJ!&?b+hixO>1KpqnJ;&Or}*Ou5y?(sCgjOd9{vEBz%;63yq~CB1(J#2)edU=Y3|_Fux>NmQMYQZT!qvNgek4ILP7@{A zIkNDnt5@n_3+2yB*vitj_rOJ=2h^IRGTZ4@fr9?ws_J`~$L#0qGEX7z1_@zV`5IBV zSNTo9Pj1jOSkWy)*p6*wF+&FIuY~HU0`RtYluKogMP7qB5r3~FtDi2bPISOc$4tp~ za+=>(pT4>>E1CB!xW>zAW80SXMd^Cv(RW416IP=iq>7c&A&dCBVTe=Cx`R^V9ABvt zBdP#0xFKaX6@!y>r-M;$go*3cch$Z`8^`DJRTCz6D{s~+S2$QgZCk8`mu)m{Br0$R`c zkZ_)4THtO*H0OJZCfOUF(+-%eSMPd2;~QaQ*;{8Gl8KaJ4p_m|a4L#KJlfU`KlYq? z_Evsl_(VfgNAe<$n}2#3#c1{%(xvh#Td6%Ou?~@RVKJgx>@v4ec7V4sRQgBph-eG2 z3=9joN<5?vTT7tt)$@lO(?Yp_eHQbQ^ysU4JwVNMz^J^;?BCq0!PM1nDniLdsP;a2 z$Z)|l`Q7yUp3N{q8r$te-ol+QE}0B98F#w0)bYeQp?v1fOo9r*j{|8e!?Iz@{aZ@D z{ij!qCdLLDnfsK3wvv!3&tNJpoA*QqZZYG)Uh~B=R8fp==(2aRK7!|xVv+@opeR}K zbBcR6uKp2CX-!$Y<#=aC?)%*Uasa)SoH^F2q>WxdA)n?|w5k^SE!Q?Jc*DAO8MFz$ zH(i+SY@m=Aq_3DY%f;9A>4EbyjUG}@hU-J74(dqk<~0L1ZZ=;TU90S+v-yeaq56Zg zPJ{IMh?U1r%Su?W$=oSxA!+SWXi>%}wtiZIQe#G%J(_;u6*N)%DmPWt6S;%CA#kDC zwjh&U9_ExROYBM=N_#j{u#v=^SIOXIA{g8C6}{|d{skW#}APud{9DWmep#TS-}f*DGH+S(~(99@?bib zZ_=xN)l~7j(otwLC*-PK`Z4Ff8D8WGjNF2q@g}DuB*Z%b?wcE!=H%fA!O^=|4$pSXEHvqzx_!x_-~01dB!i|q|VfVNh<$ezUr6_{iEIHTNKp#)MEhx O9vDLlgC;$9{C@zIrc|#0 literal 0 HcmV?d00001 diff --git a/docs/images/design1.png b/docs/images/design1.png new file mode 100644 index 0000000000000000000000000000000000000000..a31a3d07e2bb1cb9afae29ddc587262f85966650 GIT binary patch literal 35285 zcmdR$gLfs-(&%H`=EUa2nArBjwrx#3;l#Evv2EM7ZM~C=@80$B2fVCxa=Lf#+Gtem zs`~W_lb02PhrxjX0s?}U5EoVi0s^T5Y?niU0RFE)&By@0fE^XZegIWY;2i^YpzXvp z9D#sfQNKRGK&k0i01*yzWpyWY8EGyf0=o{@wPhKPuW$HB;iOHo+# zU*dp0UJ^4WCp#_%23J>CdRG>DTL)7HCQeRH21aHEW@b7-4LV168z+4?IvYpQe>U>( zc7%-`4IRwwoXl-)h`!p@H?Vbf;w2&Z>ga#|{y9!#H}n7WWaId+X#pn4@O6ZNiJp<+ z|0d>SZu0*S`#SPZv470#pWX3%HO3`x?q+PIA#84KY~u(R8XwblW}bg^^M4%q&p`i6 zQvE-Y>>S_!Tk^k-{O^)NwpO+dN_P5&#(Yfw68W#9|Em2pZZ1V*M_ViBuk)s2WA4NU z7~cP__+L^y3}5H=zs~hPFXTT;0aua_hKJ#Q?g~B_DXg_5ARqxC31LBHH{i2%$aFH1 zdWfrxWuW3vXe5aUXz5A6?UYO~?_Z_^PtxBUC$4!>p(qc^jRd7bIZc@k{a{G2GsB1i zs?87HTRT&BE;24!A1B5T%2w~S!cUYAy$<=&#S_$3vsb>a};#jz;s8T);wOJWSGyDs zq@ENer(l(&h)$pCe!jpmG;N76_*`RXS=AT~$AyKNe563-L`JJ|*i#jEzMtD_dRJI# z!}_F&<7P0GsZ|}6Nzpg&Wd|a(tjt$F@PFR&-}A1MPm9|&Y(mZTw_dcIk`i2p-Z40P z_>OH6(5VPL?#0XTeLU^jy40rezHK9RzRXwI^qAY)()hp);oUbhwx7SK5H=iEBVw2c0%QG#@2$@hQ$qAzoNMS@$eDMJ z1{I~kK$`gAMwe=lby(SakH!pVvJahV3Qv2si?eZZ=9XWspW`{Ad(3x^2I*}hsi&r7o5c;f5%XFfWOT3N10&Jp!kFQ&^+ zpI$Pl@0_>gj-lm0!Oiy`9?bdAu1JCT^e??lf#=Cl(7(4_%M1=Ay=~vIs5R4wHP(g- ztonrPuN5nqe3ylpJ2p)(s9CjX6v`QEx7$+~jAYPxX&SuDP<=cg^%uM9lkllB|s99wYUVomTxZgi>0Gd`iQjedtn-~H*J zW<>ofvJyR$cg3l{^@=J^`$h(gD_8Y3DHsq&A|;GBI2LZ+mX4YYnr|GsbHyrUXO&j+ ziw9U2;oWb6NlnSMZZ$%`F@~8!QeBweIp=`}2EtUN%w^SC`7{{fH0@S~49ENwkGnuz z;$?33a$DXLY1x9PcV-IaFB;~D*so4#OXHaP)D|fe+Zv6+9P@xd-sxmk15rO0FPM|m zD8K@Bk)mlnD{FMi^6Ims-IP^+@Hx&X^-)@Jc!N88Qi>Z`2KZZnE} zB*#=4gv(o*{nhF4$1q<+D(Rk3GcuB&?N^x+QeF#dCwkHPr)X+s2EqJ%arq-|R67TA ztbU3q%XG8ufZ@fEct#hGClPPl@LKuPA^+#|W9Mf(ltVA>hTUmJW8oO1xVnUNis?Nr zM0&TGw5rmD(~*&yGj~*+rjXlDT31A#+my+m(RHuzzt0D0o5!-Q9{n^r+mT#3PRbRu z=?Eq&-#%`VI@{(|wYtU1dlc6rv9eD*7F#zc=`hg zTFQN-_uy>$Gm6dur`ltB59hBGMl`3fHMoY+3JS)HBh;>8yy3$VhF1i+pE7JXPBnVz zuq>i%RMRlqd>$}4_;~%6N=fmZCZ)7FHZKKf6>$?$s zFv}7V)CFMF(L~zcEMhDrBA4^*>-je3Y**p7VnUVcmKj18@x~2gs4oQs{PUzQs>K+d zJHGBb5F^>EPVog598npovUN}z%sNpWVe=lQ(>qLEw3*|mWzWe{T#;0Wp}vw~qBL$L35i$r4x<#|7kOYPW!OCI<*TMKmE|bm!?}@_4wPUd z=M`BH^_18)q^rB#g6b@|p!aJE@mUK6Lexh|%uYcMNmcbj!&55d*K2CdmQn>u+^;D# zPt-CTwsC$l8#FI7AE;706sFcDvkf%MyZu(SX5W?s4Wc zCVlfmXui|g74c*f_hY@mWS3L0W zk0dIkxq`A0&|?sm=*R5#+OvMG%Q2tV)y;;%*OkSNha&#>VcE%{s-nWOk%8bT%Q2;0 zz^-Iy2^_P60a0#XAJ)0vP_}kxgo)f36Z=?W67>0K;OFqQ*VB6R(v|i2_5Vn9>KVtbS+0bnJ?8s(r zp5_Ue&thP{htRB@wZ6{xlue>7*ded_?-M8H$?>?L4z>>z|klpe;hWq2vp|mu`dH z1g}ejI3mAVGXF-Qsk$s1C*?)+d3?SYrU%jrbuE5d8r7&w^l$mR@zv_nc-KWuA zwjab7`(eS03kB2Ykc;WPe}G0!rt-4I|FO&O1)DH07D`Hmpw~@Nmi0~EB;*Xl(XbT{ z+MSUFU7uqol$2vM)EXCg{es*K3$WmB->n!&M9 zWAr9)tRcKa{GjEFux{uN7qg;pxOcdCvoY5!zn(*D)eQAhQWjAUv2Ms=fHHpXyenV` zjuZSVJLaqSYmC3c?_Ti6QukG}LIAsjuAS6)4`P%C5rTe6( z$6J+J&{CDpeOU_m)$77CD!$g6=h`m80`2DWgsaP@x2qGP^Bp0cRQMq8g*^;KhSQAi z$DuEZ)4^>*5fAu&f|9KJ+b*GQU9ksX+3Vf9x_*(m*hU^>7|n};0Ltif9ii(rU#8=H zQd0C{w0s+!w)u7diDw6-q_PdI1>-PO=L=9o`g^UAPX!4H}scWkb{{kBTQMnAH z*e6|UMMCoJb%rLSOJHCC<7Ky(lxpNLDOFjPb=NjX$Y6n*d*fT^Jt{0D+B8tm(GBbs z=MrpKN==@R2OeGudQ!UT^iIcuExXfnm$|aXoYk>Z^ zfE4inje$Ip^O%XZamvGmF5K|B5;aoz7WWbxE$4lr)Oy~KcBGo0J7$vGDB+WzN`J@e1iss0k~wQYnP}Q%80V>>&fRbAl!`=sQQ>%;eoSVyW-wct0qaCYt zzWhPNE(Rs3QEF>ElaJ{$sX05Vo%}cwl z^!;=tJ+IArKV$HHma?L^4qFD&{aG7WZ{fZLN@zAaY9$OIPpw9N!~!EY?)V0Hm{;mZ z5wbM1#0Ey8-+CCe7~1a?>9#Rt?xl|9J5*){rwbd<09MgH@Ogvm}dx@!e0cP?Pz7_MHVnz=?Ff*gzi-eS3 zFFciADSYe$7dFl?;J1#4r{J+*X-o>kFGA7-r6LLac|?*b)BMw~@sFGtQe-e4)9)ss z;7jjI&katbA~X9oYs{9zOwYLbHjdp$mfu__l&_bR{ql8&dIFsCYDl9SH;IiRdvRrg?dAilS&-eqe-LVelo);pxpD>oa7YsU6G7I{AUWsl zJro243h+c~WH?WLy8J8t6Gc$$yPuotNVck+HasRp9`lczdA;rk#~nnbMZ)!INrsc5 z@sD4C3^wm9xBaT?WL4I6q$wB$QBTGCr~6?BMYcNFQ;sVq)!&%MB}2q5Dp-2<>#(ZQ zblwPVXra&_`Aw*MDE3g2Ze1v0<-O2wAS86#kC~*PNKD+5zk^4Fu^9He0cUK0cDn|0 zqSe1)%I!x{ZbD#ux*OK7F~|&fKXLB;4#qk(lupiwO{$1OtnI5!lwYoRPg<;;oE+zM zvu$MA>?WK|Q|x|%L=h91_F`|v5*|<@=(6BRNcjmF^iU7r!v0WdGvHIHyrV)kKZ+)# zej82GvB_T%<)XdK1l^%_MKNH@Ez{h(e~hm~G9DsZUp(9q=MSGw_P7gYG88}e9N+J~ zbe;8Q8M^;IFvbCA{v+LX-|OEm>*u<$PhJ@75hJJO`l&lfAU>4c2VEDVQI|4?50|!t zj|dgNBz*u9l1zeBEFuAhoP1_d@bgF4?mLkGPsR=C894PrUL!uHvwB6m0VSnT|mkARhJQh*8| z>u1!myyQZ9eAiBvssy4&fJXL_$U#hLti;FCoHNtH0B(yfKs81aZUi}fKLGWu*^mN7 zxV^{L)r+@OasMTV_gDE0)WF?8R|2g$5>+9grWtNM8Xx+KQb5_BMlOc~=*ZG*EKmT+ z%}a8sfLak{$uGOQdTAE@pCLcSbN>?T885ZMh%qUwlfiN0i({D6Y0#7w%Pci+F7Q0{*UCl1KVkB{Ik+k@yPJVc)w2_uPoBv;^_*a(J?DNKl@5rnG> z3LZSy>dznwo^Azy%tI=ap90%SyM57=%|TStC?ea)2WTkXSG(Uxh+mw+k``DOz5@bU zQNdC$KAZx3#-JLAv-Z@ZOpxIi``vn{J>l38PUfZX4 zRT$kBj0l7}kwV)l{h2EzyhpE34HWB0KT$@hy&T(%J=Nl8iA)$-6_|RrsGb(GbVKOP!?x2iY((rJ!!ToYmo#T}Jab;F3Gu8d{ zJTEG|LZsfdR*Q<@7zXy(|BTq#A7~rjg<_nC7&!)0ao*`Lk7 zor7EE)q&-+`7pCgPFu1>63UUPj!ehT+5E8ial>;KWMTC*+I`GWn6SHmQba8=guc(u zKqrz)QajPX30Y91!>ieC4_($Gvds`tB1lK z{U;-A*?z9Xh1^68djr(OyyVK%HI*KTs4d`d`vuYG3KTAs{5c3QcBA!>pgm6u-i`jM zdD5HSvl9&V@_9Al65wT$1WbTS!3OPxD#Y%Z>W;+*SbcMolilJzOm`%DJ($uJXK0AM z_wcN%26Ke=g4PNxpaU%6CB^Tr+ZNH8vbXg8Vh%YyK~&2X|FE~rJp?hL@lcv4@sy~u zN#{7T0cnE7rsI?}VwcMAVW?(idjZ@e`GKu^SY6$rU<8kG0UVEhND%X{8@_;m@ZQMM3^EYoQ}-o1HXYZuKi%xh}} zf`NzbR$S@o_!`*j_&m(DJxZO{iqGM#)?sa&8)euUPF159miTT*jLq?-A^qtsDaIh$ zh7QD#!^2}?+E-3(GEOdH1t1s~lWnqK8om<}%_y?5ogOc*5$&^^AMcnXY0mdvTGtPo ztL`>yoIe@(dK>h|6-4ZDjJ40g+a6pN-<|^J<+|!Y)wDk$M0(Q_VJzX9AYV~mva`^{(-TO4l-0$oo0@+d>Bwaxi8fVl>A73&8Y$1k2)-gB{Ll2KfN-gojnAotiA{ zs|tTaaPJAhzD4BFv)Q?&3>U7|WyeY0cGT@4G3!*!yi3>4*XN5Kl3r?w#Zm(}`%U_T zG7&;}&GG95OfIh~2}uTnt0sS#U+ApHA^ZYQ3h1)UqY3l5hV*Ytdft1Sk&IdjEl%e1 z5!ZXgElmOSW8}rd4haf^9tAVgeK`7j_v zcK4E&$D=w()Uy0l0|$f8%7|@WVZ#em=9wxpb}^Rs#)s*QW`=Xf&(=?rs_{~$YubL=D5=^}V!*@=4p~cxLxhEtnNO$>y4SF;I z;l=J7p|#e-w@?KKed0qNk_`n#jLNnYDM4Z!T4-fh-u0tqG7IOXCHFh=m8bm%W>_jx z{2tREYjb40hGts7gn&51iMkRVs{lSCk#;TEXc-p{#}sA=$eWS2tw0cbc@8>QByHx# zP%y^781gAiQ71+<6YycwH(B56C}4^M0WF?5x^$l6KWja1|ki&E zMDYZZM)R!fq`3?95XtF_FX^ybuh|b~B+{?%MY0r%O4Z%nz-iE%w0*vZfUbF7dWWFl zWm+F**bc-4qA+u|(Qpjv9J5@7Cupbs8cs09Gm6Ggb}mc$?g~tgJ=-CK(2kw&nc~ys zt$E4U$8x?HD}e-eF+Z^DyWmRpS}!~xNv|mQpO=d{K%e+urcu`#iZVSk>zXo@u!v?< zDr^xGmifZR9*VMjS~$+X`^qyv7r_)sb&K0j94C2A1^c=SNczzvKt0Ux;#~3cq=l7G z*MJ5vfG9(2iQp+vvr|4w6i@>zqp$W5Js$W!CEwmp9aGD zH)4xb`Be}y>cD|Y0%NR7$f<4LW-FqaP;#%Dk4Hv>_6z0g&-ft9wnCb7ywr(Tj{KY; zl!ENf3POShupwjH9t>{5oYubbNG$GNX{i!L6EB9L7aGeTxx~QEF~X$#%#%_Tf+7xO z8|75h$#Tg&=Fp0OjGyNCx&`wr&yY$xtbaujjRG%oBR<&G^Q#gQ&~URQrf8jOu=g>4 z%D=-yX~%3wuz1fwGH;;}mC&xx(lc~Cp~#a~xm$t@L@GYA$M4zg(RGWGuxWrpzC(p=~=<1+VdJ{nQBfe9Gn2} zHm@0D`*Y7j+x${JK8o~Gf^4C|`SZg`+B2iZpE_aj^f9PU@T z<|WpPu})Hr7y=BD?K{CJLr4WOQ6+)H~(!V$Gl1;bE19c z=8qV@dn6wYbdgL|JLZ^@&*Lj;G)F2)$>Rr(PVn`v$>Ty};>U5culr-`25$N`G7N=M z3UVOB=O7GsO|l3B)Ft(4OL7<#Ui+0cK zJGWm^z*$SuH6=osQoV3Rn(jyXfcslsfJ)S~vWpOzTG}SX-18C^6br0HFEhAUW#q6D3zeu;hzbY;16n_+ER$+lu3Q~!tf}S_ zR_2HMU&VLA0-<_-mw!Vk&7ff@Rs6|BwC%-GCE4q1B(ZTxz>FpL>6tL?(*46>G(pfk z_sr*BM$L!3bUJNp zazoojBJCVo^B+oPsD0Ay7&g7TV0!}uHY65hGePsnb!ui_B#T3e3;~qnwj3@KfbF!l5gV$-PPuj##vH zbSsD==_W-+;AizWJ`;P!IMLgD{|ui|m|p5)o?lo5Fd)|kLi8b+m%HBx;tjOmdO@=$ zy%Sak`QMCJ%Az0;)in<|V2g%p8kRWQcu)6&bP?~Jnb}#)7|Lc8-J7ir2!3mNxN|(@ zG`%ps(uV{TUyPI%92<~$Xz)dJ<2iRjlS@+NyU@F9E`B#LS=?(*o*r&+&z3%Gha)CO z%5F_S@k2*p7Gl^(aSc$SM{N+#nM0kPb7}K?+)2PvtBVQOqm2C#hA|ubx9uu~o*|t~ zdAn}`7Y2|5{$WuS9>5~4e=_DvZpkF85?RQ1FcJ@@sZwGLEXzZ6rDpY`PFPVD-9Q6g z!@``pIBqZ!h&pf75?%#TVCq0#8JteoopLx3)!+4^4nF-)Ows~aR zf=uE`36aAq{rUkSI2XQ??!)o+XL{9Yitvhx8td@7Sol=h%@oTadn@aW|tEzTAHUHCI(9 z{tLDKzVi^S=O_PQW&XwByznp{(E*lD7Y^@kdUcHj3(@fS7GILJw*rkhsTrGGCGZoV z>+&00E%_|P|KV|Ht07xz{lQ(Hq@C9TtP170B>&wVZ82nvd|md%MS}CcupB^F@Kd1c z7J8s(2^0DMCV!52sF6gO2?#thT5xA1~(p%i3VM zK)Ddn5S?)Icfu>H?cK9+UvSlYT`SK2-NXba3-440{Mg+vh*AsqG6;8@(&E3_qj!R% z9}1mfEvNrAof>z&DGNOZ2L}M7!^6zPltW83sy!f6qTTLsf7XU^jnni4!75Y%jl`qW zq{z5Xd1@v-xA^jJGLw?mLV`+}stCT=LPxr2spe#@&_38=xrEr2IvsOaITlik&eCw$ zNLxpn;*5cQHjEw__j=cS84YT@uDJ=UEUAxPL@fBTWSxR!A1FE@x$T@*PQ&tYehnu3 zK1?>eJ`t~C{~Wos&~!4g)Sf1nv9z%&9iP`mKD;J&NJ$o(kNu*uD_SSpv2~tELnP}! z!cALU(uTBCe{{i=iK3V;5MqK3C5sE?Fw6cxWKT1YDqS0lARWB4(`i(4`*mfKX<@9P zH6I8QpH2mO4zb4*`l?Uh7IaFBJ@_Xi9N^Xq4`a%SVH0C9)ZnPfEi+=hp^uy zWZAU&`mp6vp7k{hdi~JQ?)Kmi8%Kr_B^71B^M0XZOOGRu9-P_1%wDg+2EUseJm!r3 zB|#9(^M})y+ z8j!kh_M?ontuHD0p+=&H&%+75?Ya3%0J-r+fGz1nn;F;^MX?bMU3DH<9^-CY{eop^ zy8>r$>vpd4)ZUmHJy(ZkD^smn0^md{N2!1~;60|U*R>QN`SayJ*P2x&s(SOVy9|>h zwOu$@wGOaVUIUm1FRSSthzd|~y7T#dLO?Povg?bWN~P*9X8rwp(iz}hoG=aIImV~0 z?EzBjHzR*&#yO40;20|!VZdtSyj-P z=(0h{!Z95UQ$evpINotN-*q3vmFJHcfG0bw7siBjt(CI#Os|Hrz>;|RDGo%9GD{W< zO-mh@3f0W; zPdHHW$J$*L#yOS_h*!~EQ=zrpBNQ0J4-&+@Nutf##q z4YyB79;k^k9IY|&?FZMU*`=pA!p}X^Q)OPBUTDJ4KL0B2rvxRr2=Oz#gO5wMg)&GJ z+2grjjwGmh9cvxJEprFKi&ol7ETk39XzS4j8g6P>n4?e$dUF*FzSE)3(jbq8TQb;g++&3N8{BR01ohq^`W6#6s|l71dhO*7?7(K)Kgxsaf9 zW5ZJ9fZW;u_J{|~pxBHCefCK+51CYrhX$~+IyEuedC?>(^En@Ae#_IeYzO=<1wnE1 zl83^1@&LHD8O%S%j#3`Nt^)LrWQfwOuB>LiK?-6a^vyqt3lm$3pawFTI8X9fmu!+f z5=&|=5A?1oT^0x!9Y*F#UGH8soPTT@CuujVSkMfGimLUDm!n$`1!$_t^~^t~_sw}^ z%LZFLmUwmpxj1lc6=I*G^1{0VrxY{}+BFax42>PM4hWP@B zk*1CN$`B>n;0c=~z0S9N23r6@Vy=TbC+;M#i5xh|d6=HmerAm5eI%51lThRal^Xkv zA<)Fv*KJaIXuO5NT4z(C&&Ygs3=#C)+Qzd@|a znUA-_&B#+PU}SBa5uCgt#jdW_V|a$zttv&m)??fPLpF^|nI)pcXgg4o^*~vV4E-Yq z?H4g@O%VJOv02H3tTeL~tLMs-AqCz2j4>G^qXUM^2?D`wgD_pINo2-v?R4@eGt+fg z)suM-gKL!8eRTI;LjW7$!y5+-s%vmwI=>-^-K@J6Kklc|FdSE0w$7p1X{0>#9M1ca zM9S7Y{-Nq2B@(PrOd=+SiO@dd+(rTAu-rEmubCEe!fkD5?uLJ zG;XZi`5dH}rrNT!>#Dd^+O|V$wGmTAq)oa{IJT)As8j=zvZkrg^Z~bu<1=qJv)S`N zP~1RFVW@l{U=e+gI+L{Z%?W zNmiEOnHMNspBdN`jUL1*sS$9vJSk-VVCOo|&+aa=ZL7#S#}(!P~_ryMk)tv*?oHLV1g+sk8kn4X&ZNi zAv}A;4prAH`XBp%9Q6)n=hK%>Mvit1a+O?@nNcx2c~P~ty9rc(9#QL1l&>{U%CaB8 zc5Ru>-~$-6K@lr?%6-0g%`CL0$%t3V>XZjrhtTikv+S!@jsOth5s+lx&=0})5bT|m z8fHS^aun=88fEC@c|;%mIN$sjhusj-)*-T!=+oLv^b{m5_9N8JajNJKXWmN>xhwr$ zxp!0p&l29)L3m#*)_z}MJ5GGEe(B>n?!GdcX>#pPcNuX1V>~C_-9nNZ50_7Fbufc2 ztxR;VUZw%GD&RR4dfypXI`|~T>f1fEEv+_B$55ulJ&vGfh8>F?`N{iX0*0~NWZ9DO zo0;MKkt8_PK8_bF{8T7Qq&xaqu4Jqflg%%(Q=-0!J*%>I%=iUj4kKsdj6_Wk8EM)3 z>6kcPD|GSsE(=>N=ZifOZ{8{6ai3%hlVBc==W#kEx*WzylXIQ!@{DwUGZU+b`jaw! zP7jiTfvKhHC9X-bgCE?DMis`V$6nl z#`KPE2A!P5@th$>9yJR0`g7cy&$2cA^#@m;>!&&aqZ8bhUVu>iMdPN=wT}OZGdU}j zKC?X-E}n#-imn$ll;Ko_1aKc-nFl&?z{$M&2=e8uPK;Dv-zF@|FOb|Ww?06S{;?ha zw|sGH22iPT8bD}XGLKsQk! zL{uArRp1=FIg1HMdBPWHHW6>`^W!IaVw20H=CSxglbtFq^|bq!G8`K$uzZRJ>ji)^ zdJ-N#8;mOw72Y%vp!a&aZAfutN^NUbHfJoGFx@Z9Q)z`Zfs#NFS7wHB=>;oi-yF6j<4zXR*}m4 z5@DtysuW)h&TkyBH@e@K>Eh$g;Bg|!20Z}yt%2Qw{m~?vTEeu_FPbZpF_;sFg;xF} zC7zz(CQG&9cZdsOoxfHun`%8N+ag*_vKsxN4wPQ)f-k{Ij1T4d_2led6R8j@h6dJV zY-8op<3U>D6l{_*J!Ym)CvyVcsRagTf3%e1{Uv5+3Z0fZ+y=yl9CNI(6BGi+Ci$J5 zh5)I*JyBD;;#v+oZ>yUh4NI47%p4oUsPc_V1`&X}cLLhzO2mwN`?2ki_aO>QBxN_6 zf5VN42GjD3^O!nfhEVr?1Na2CaMj$O0KG(o5Uvq$&`a>Z$p35mQ9JXgk0QWfZsG^^ z5Hrfk=I4j&D2^D8j{jTL=Lyge@# z$ma)ePCP}=Qz3jt#(si*{pG5=U$0*9-IvJKBU~v_|8{!nzwVE+aUzAoz)MHC#b|5_ zrk>=y8-cg61C8oD%8j5!HVB-^o(S0eVt4^ar0>UV)@uby<`-@)m>B-$Prab@Xl>kJ zc=x`30RUsm~?RvHyZ%6!Izjf7++_W&_ywa zB9_$1#c3DiE}&wb&jK|3BB0T>D9b?eQod!O)|%>bqddSYHUpWcydeBVaDlr;uE24Kb0)Pxv0#$2l);ILe()T3U z`KMx~Wd|8llTY8|b~6zXsz@;W5}mHtzSZPI{A!u$Ujm!OIJhB`HN)-t1j_gGkwyr0 zQLhJ$9XgZZ^KC&*#7(@M%G@L2#s?wanG(oKmnZ`DQgF@vH}Pg2OPspV!609izR$9T72VG7zu z5>HpJ5{I_qTv{TD4DOhmE=q(s@DU8j-~y64s2h!xkki2!xTz)JjVe zXy*DV2%9$V*EI>j6AH6nwpQABqwcSw0aG0m3hMX%xT%XxG4G&AZz$?Sg)5QB&~_;! zDr(T%i=`6hko1q&?}m7{TzkJ|al5Ci6pAeK82Jo2)x>g}5~OWn6gQ<#f|KgA8B6xk zHfFImu5M8oDoqq3$h}aePehA&doD8OFEDk?8f*O=yfKJ3g#;IjMAG%j2_c-Gz6U_| zRNM~9M(-1iZuxp}U6>P!T_y58=^!6}CykQvX?<+D<5UFfowiv?9@+q0;>{d4QO=UG zR#m=;G|CEo`k*-65ffrU5Iy31B+CR5O_%opOO`&UlwE0f28?%!Gn+h8g+1j}c0aOk zn>@+)sV$4hOGNkZUG*)i&6Z&t>H%{{&$H0q>$JxwHa_5Mg6B4TtfNJJQoampZFU{ElL<>6>0u*Sw+oD4KP_wxSgjoCkP z)W4MG+e&d|8~h_)h%{vvJ`cJHkY>&P4CLn0M~3s0ZNM%~MNUi1`sJ;#OnwlqU-5+z zpFIY6f2efeu|=(HD5wBxmA~JvbQxRna0*2E{?+IqoD$p>`x4_f0zp%LHjp{P89{46B8w8v!U(Z(^k!1Z2Zfz)KId z&0v=C))hoc(B}k)o=q{ljI#B}vAy|~ALbl^b|)EqJYoWJJrEk!VUjR$A(yJ`nFvsY zk)~lo#n2oo%OOhNWSePmfKi;nuy9dUc<5cSmT>kBJ2lw|+03)EKv7Pvf}jNHQxZ8L zQ3<6(;iu47HPj>mHzEetP#y_|ew;hNh<0Y&Qt|?nf?+G zg4Hv8=(^jsWW%rcY2NtV`9~|$tC9bz4M@>+P!`oXZ3RORB5M5_mq0ZlxE9+VIcYsr zW6P6s{%dg*WR~Y$|2$P-CS=H9@rH;cv~TvjIIaNBn_F>m5ZHrmT68YNd>S2Tj3e+; znfqx>J_5_oOjv>Jq=35Ks(1HYaJ){w+UT0L%fj&aW*?B9Z+E?!T9ydb3y-&oge2>a zOlZ$KVrn5plAGrpy*d-HRRs)WR}Hny5*BVXFwOVS7wb;BamOL>Ro$WU zt*yj&x2RBE8%;^p==U$+-U{KP{#d#*MG%C&K~rv(iDI+YJU*q$TyAhJ40j6psmOgE zGyT%I;iPl?o#WKZ>UD$5L^+WI^(yww?Y=d}rYR*6F`v~It5EAZnaBSA+8cDp@{32d z#P}buT^bh8i`Ws++B%Agj=z^y5^uSM?uaHS?+epx<0dY1%WiswWwWqH1ChYHf%O1h zDsZA2)oHo_4w|2OSB9%9-Lt0a*0MM6PH!OX@dga@?UxJ$BC6-|mbit8fi2vgchh3q z0h`ap;NyGpyh){!U!WpEf|`r8To_gU`xA?#Hg(X^(GM*zMGESit6G8D zZZJYqJebw}_onRU!?R<4gKcKVbRltV&#DY~3bbNWbjR&#DY?rAotNM+EXFRY{gMF^ z-C@%f%i%1-l|>{ZSPs|)=5Xv)4sLZ-ND#>ca&8E9t2V7vwbbqc3A0s4;8CO8P44s5($GB=tQIMhf&LLqetW3h z-kRcwcY3o`FJgOWUr6Y2eca%jlPjsRV6p}gZhEq0o0)l##kWM&1duVHiMh}jC}zK8 zd$R8vM1uz!b>F|M*?LCB1ADBNVr4s<6GJ+CxiO9_Ga(Dy=qPSiqC&N)PUP-e4?Ejm2wXdc0gXYuQK&Qgd3_*) z0tW_Si{IZdfI-3(;qv#I*Yz+yHh!?6L!zn2G5@?#pb~T9V2GwdU`-csyY_Dml@%}V z`FL6E)B(IHBXCtMB&bLO2{rEe>#fW$nUmECE&AJ5t8fykgs+>e!Wm{ivNQVImn-2T zFf|&OzoVoAge^gRF2?X)3WMAcgTh1ox{6mRUPUd~Yn@SVsR-mALEl#R^?x|{eC{>**Xmd7qu}cbSvt;3^V2vo9~3Y4^QcyfUW1bE z)^eF+ATCUE!>K(u$1M%++Im==w4X>XFy(*a7WeMB-!w@nzV9|Zb3T$&!tL&=WpW^x zX&TNu*{WVb>xZh`7h85jubiJ*AN`rl>y_WQ{l$KjUNzTv`={2fN?lEd5%CE8HuB;j z7NEP@?+$C~GfTyG^KCcZAT}>rFE8%6P&4$7r>nCkerO*D3MGXO4EJhPp`O%WikqvAb ze~MOr@4R_9Ehv;0>}%pC5c4lyK>5Y3sJib6deLz#V;g?dF0(2BBDw=u;u5cS6b{h| zhnpJ0r#r4`Ot<0cZb z$bfP2;F9g*U|h%QpS{<5*0UU)wA?7wkzbSGX zUb;nJVF?-#n`Rc6%VX8kT3dMWZCpS7@ z_}>$*<_EtoPJa8^(eaDo>MS+zkM#h5^xw*hCVHcvuMx{Cz=QhEueWg#o^SWVs=&Vv z`P}o*Ijf{)7^j~wRqOHxP~d?l^h1eA`wJr1{$-X>p03YsbV1ntHtgr-9}F6HHeJ_Z z7JI+knLq0F%XqzW|NKJ$wmi?W4w_8FD;|f?Z*t$bhI7Ue3oa|gQ3CcV%k%OHM(V0H zHSCT-VN2Gw+)I4pd1f*ZJdAQ7F0>{MDj2$3#gqH>`_!I{hluE~APn|#D&j;K;OCme z3k;PNQP7-&y@6frTJ(Sy(MAt_w6$ zhDx6nN)CjGJQItk_H=uiDT88o4OEW}JQ1{;Nv!wdfLg4g#hjRau(t$ODXm5?P2#sW zEoYr1-=ah9>41KpxyOcKAWXts`#$GWnp^()Jx-?Isn&cYB4=Qwc9l z8-K1t{cHH`-s*lreb=qYWWe&vZ%7jNy$2|TKNu`V7$Mzf&1$AKa*@1)tpvfFebBf| zup7}h8V+}vUOB3IzM0H=-98#`UAeJ1dQXl4@l2YvhK*i+Hg}%dcWuGXZb!HNKp8vT zp(oC`gA`D{MAEsw`*$~%*@3;Bz+>t*qiz1u3aax8E;qvci%@gbi-o1_K>d4=`kP!l zIys$P%hyaS%Mb;XoY!q9@|#@0REyjv?n>epvl36{*Vj+`*;TdgEm~$E+|a=!|4aJB zRpTLDS#8fS2TOK&MUoqVcB<|luR7l{6TveJidmUyYj@L~`}{8eMB)4PYx8G5(TpIW zmUaD?y~#?;?&-WshVjaQe`&pNM2y_nEE&dHU=jbmorqb&;?Z4 zOH&sR5hXOjgz%cii+<7+mYr%!eCL+~0PP>g{TR=CM{-gkmipqB%gRBJsi$G&8ZFuT z$Ts5FtHi>&%lb1JC0L-=C)DWc_HL@;GfFeCUrz%7T%Q9uL};^(wK`WnLPbLVr5Nm7 z$IjKVz>i26qOX6i^z2=j5=l?bORko;$g%2$IOkiE?=2>3)P%@&uBwgR)(X@GpZ2%9 zyG}zQ7A=*|$6(&qU7sI*OhC0>?InIRMC#l*=jbkSk8-A*cJ=c2qHff*_p>;}k$m)_ z>hX#>nme6s-A_L&VAw2_lZLHyonw! zXs)Zvzm_g5RRb9=M;hEEzUkTI>J4liXvM?MC!SzY%&w0np!r%|U4(Tf zur)tVf77Lcq8Yu0kVK@z3a?;pfJRQkN_EDma2}^Zg2uFZq)^R4!>+olmUGvVLQRD5 znQW!SMc1z)C+`iX(JTTeqa~5XN*NqEf*{2JEBMQvA3gVJ?9-35?+C7?Y1vrQ9Y;ON5s@@JI8bTJslVYaBl4$l zn%dS}8TgWR8>P~~7t$@+gWA_`T4TBYO!l`v>@7<5W)He!D)lFW`B1kVXzKa?^Ju9y z*qDjI>`M(uo{_*4scikg#ME>0f3uk2UiG`N^SM$h{p(>_VE>8f*?Nfza-Gbx!28pn zTIZPqouQVn{Wh`0X(DEL=4X%iH*kaX(QJeT@fPRW663>dah7>kvj8VOT-YiVSCNlv zRZE7DwW=~*p==agGA1rL zc%eyqCXZ>Uw4l+bb_=oklSEQ=U+1cEgeGAW{8AGM30VS+(ILP&xQD}yepBh1DJUNx8BFIeJ99%5#;z%)PE=%P$p zR%PoOFUYrXiO)L)1!qUS%t&lhlY`s`@6{xNh@FrF2SeF#er^l1+cI2Qi;0POvc+1X z(`0$IJM`QMqGl=MnGVC*tG-<>>=(m9j59hTh&dJ28x=cxZa1p=J~`u1bZDn9-^C{d z;pVGabF@?jmcz8pb^F}e`s>D@F+1Q9&u*}lx+BPq`#NR@qXY z-%usB(6;Gp>^S7KG9|4!>W#m8E)XV0!+&VQd(|-g`$Wv_nE?Z1h4Uu1cgj^A9$Zrv zL)8;-+)_0~OM`3#Y@Z33ss#**8kxo*k)b>>``gSXj-duUR!)BxC14fzxGb4ey)G z+U4(-VT~Wmf~my~sUW=Te7%ooR+(FS%r1}ltW~SqcQOo>@DBE;5dRIU+`Gu9s)$RaPD7U6lz)^!1R`( z{NXz6Tbii4_Yy0inUI}KSoEjDe`F~O6*f-wbG6{5ZtF6`MsaGkJYn`A{elqgUF6Pt z{IYp=39(|WO4z9M@>k@$>3dp2%W?ptLV=H=vBlOo*B|~}`tq)pGNL)Uw33j2_QuY? zHF^LuRsdjo0Wvit zJwPVL59Aj)nJ)vD$lP`Tg0oRuQtowNH=t}TNUL=u{Lt~cW7B}y$&q`)%_koUTDkcIwe!Yu+@H5;jj;j7o!if@roq&IbN#sY{~q2TXrX&*XL^Y zyS4cNC((Wa+;5Jp>+zx(4`?k^wwmVJO9gSEkk#v7)8odxZF&*XxO;rbrWJ>HEN$7uX)3ZB{x@#g=Abusb{Qp13n+ zJC=b$+3_Us)_W&J!&jEys=QKgw3Xrhz5bc+=NA@P3NhiG5YBJ)*uTCwNgI(6%Sc<< z|K2dXi%llwY2GJ8!9zV8s0tKTApSca0!06AXBa$SNvU2*9B5iG2Ds%0cCI-`a#`K7&pDpK#sa`G~urlL6bkFtWbp}8gQOtb7gS0KB zrP_&nVaXn3ePBCj$x)l{Y7p95B!lKY0std=E{WDkxgjM9{8Ai1Jl+VYhkFLX=uDA! zN5UHL>|JtuoRyBic>A@S)qUOWEO@^?YHBF|@*Bw2WJ96~0yTSBEo@(J!>@wwvat;M z0}+-F5HjXE29u!efXdh@NVO5V55MZHK*iW5EjFK~nF@mBjiF2L4d=bhD4y|haA{BJ z)|>^zqsm%U52Ga~`Kif2e~}a?FPnXR{-}S#YbVUMsHUz@m~7f0_*uqfRoaf2Q}%ZI zWaR7n=ZZn?jA`SPYpGJC=PLhl)vRSKH@U5)&A>%N9NOIT`#E;yM*i@cV4POjj*kv( z_UP_rOki56KtnFs&T+!Cx{bN-8WsejKy3@mPSkn0Qg>@DKZbKQ`qKBdc;a7U9LDRP zPNK3Y_|ldGY}6Vm_)FMPsOV`;J>6U6-MP*-AYn{~O}OqBWqAyo`=Ga;m)imWGeCae z`;#@^ZI0LuyjA9dccBND@sqN4!SiCdYem<@dg=E1wPssqy456K7-Jq?mJgib)`|w| z-F>j@g@w8wFvr{r@*cp2AS6@?2A)HYWKZm)!?UIJ7oz2fv1pVE1G9^-;!_Z;;5JZ& z_Y6vlnrgFfygz-RMrScz`03Ti@j>952H%_?N`01yt}vqJuZ}Q~Ys;VN165n^q2JP- zg-@Vj`!Aw9Kd>5J^P6YK{;)^~l1c(`JoX^%c|4ZDThn0u;P335%R8I5Ta*9rSMh_*K2E4Pad#eZd!MmXUlW-y%kb_hvY;M9e$uMBEH)& zR=Yc>Ksb&>8G(V(SkeJn&y%Basvc*^$oxfMHhIMF} zsiZ||KKjm~HF~ceZ-=6SX7$?hdaogdAC#vS3>jESSh`w}hlwH+;L$m_fYfUU*GORj zn6)wMDLw@}?tlmqjc^X=4h=kuS4VH+3mVh}_JH_xz=bAY>?pvgfvoxO88CHm7#J7R(+g~Y%FvF356$fNqKe*dE;=i*k z&GKChVh$vA&z$sV7^ z_13)p=8F7NeW?r!SIZbIg3p%v3c>)sY1UsMXn+JoRDhThKwRztbiAZJ%Q%e|ucDp) zN12*iDQ>Gf4|j_RPOdr?x&#&l#+<6ipaO0_Tm(}lZ>SM@EJVOy1Ga+1B-#qvl4SC1 z#PJ55`)d*#tTMC7rRyz7z9n?OviIx#?AKpcC+`V85YzYkDAs|t?{RTL0csl3eS!H+^7!Zh#;)yecLZL#x_cBU2HBF?$vujZ* z;)k}4urY{W)?$_}!9c0x9x&Urq&tjXwYa8932Ig%u!R(M5BTjTO9vbYY^{3yHZbsM zT#va9{CEgNroJFK;PR!9l4fasQndJ#5yXkn9gIikjXoen;Ug(BwX@VQWT1KC z7htB^NoBjmn46bs@vrngj->l(hIUr_6>XJ_uA&oguQ_4{0{5b0FjRFtXStYRTT8>% zE{HA(ECa?)V_Z}%cUn`1TngHBMW4QuDB>}vPCsIyOpHwy8y%H2Pih_D{}XwamJ@#g ziX*|7C`=WI`Gd1t5hvU*Z)%Hgq@Tor3Z7NS&9*66#JFZk=T3V$HoHvtT80`eRA+(6 z+%as7r8Kod+$(^|xoLljR`M(U!S}f*^a#k=yaVP{xJamEjo4ob%4qY^$zn?BM1k;B zqKF*Rlp%D3tQ`|kpa8W@0^;6Zdg1Z5u_&ApW5sL-`WS6~-Y(w}57#0Hn zcUm-@qBIZj(IXfDGkw(pR$PF)Yf<=E6m{MkkBIee&%@^Wj+Zoc3@kB7D8<=J{nHQq zpK5bJAEKdiB;Zd1`M#zBMH>P7uUkOzFkOvgmFE(nbXgCm@A#Z9|58S>+W2+jSF?I4 zC^IT3X|G#hN=bKKBtO6G`JSsAKdTwB6Gb`~dP4-xmh8nkIrq}Hd zOTb*hbayoD4|{f)gKNTg#_S|c_{6yEgySLvPM3mATs)G_nkF@dt6qqzUKst|97oL`Kp~31 zeG=$lx|ipep2&9T1&1BHaf#9=Z5rwUn80W?4f2+BaDF-0>FA6Q9RzGOy>aJEwbTVv z1TZS|cYvXF=B2od{YGR94nS7jLlaPIboZYh`%vEj>Y?ZD{?8fY=X5s!DGmf~H1;*0 z7)dwqFGtpWfAol1g$|z%d47$>fxx{gDBqsqHrnf)egIr!;5Kf`cO`8EesZoxryM<- z0MZZ^m_wUp_|F(=Ud`VIW1vR>V$&Nj0ff~RJWX1ObOhh25%i~QTlbrhysUV!Y7Byg zVH;~%Y%Wul?J#|E%pp{mXo(>Jl)nYyqnZnId3eoMs!%UaVt=qBuO|`q|9N%N{RvdU zBv(><_jel!$}22|Nic-7dL0>zDfAnO|eu80zUXWa2Feej`(I81+Y)zBSEsV3^?CKJ)MTQ z*M)-j!BPn#xq^?^Bj5h~2RtZ7L|m!OGs6d;DLX@ol(k)ia7`wAG!1C!KEu(rmV209 zU}M2^DtK-D0em4GxmKB3mGXgCV1(?vLd>EH`erc6|9R_t*A@^UTidCA^gIE;cuf!n z&NZ}UD+G48q-mo*q5Y3Oz|naD026P54P-y2xc^vp^uESB`H4Sj$@fsDV~4_wP=78i z2M-u!+>HvLMLq)Uved^kip((Q`l^Huv&^#k1pbOXlo%W8G+?a($Ohfeo^jCD9@jUC zeZb_GzZ4{va)@OF=2H39Bf_IBeGNdZZ3bssumC$w^KHaUAhm@+Ix^aTrGq*8U&wW5 z0z&yiM?)JM1UFcFw>d%b8Qev`om=h>)TZ{fOshv%mNlZHW%-Y_CU{i9tlV&Q(-i$Z z6WP@gZ*%9vJny@yuRfue=PzKf@&lFg^BIIYmb=QB)Lhx4Rf zQ=hbQ5bj*K6(E*^MAvP90&+4$bZpZuE&uMR1!x=B3HqlA%fHSzNC@SR$~4eUQXIy` z?5k6{>WtX8u3Cc9kSp7)+Zy#g=~Pj@l*JsAyL*djO4u(3817*r8z54bTlw^Xv+`!q zNO?pdnJXrXDho!C=?kQbD!4>?;a_Aq2ba!X8S5${r~Pk`2iX81RY(O+RniM_M3Y9R zm*d?r!v>vLsQedGVg?RNo5dN}Q(Z zzXUq*tB}-`ZfPeI-6oc8HXPEjhcWT1Oo+>#!9gEAV~)i*Gy0bKiQxCH%tEIg+3i{S zR%M~LY4lrb-eF9g6cHiTE|lr9%`>>}A_MIit(eNUoghP9@X?}8lFDTg6d;#W& zmGYBlmSzT0Ie4OIWjc<5CZ}!q!28Ze=o4B9=KL5(^SSdprY}*nG#?-kmhdlt$PyEv z(I8A@vWyTdiIB=LN+=|J;k_B`OBuRg&I4${>60sG1BHG*j+QzYUUE~?D8AK(W#tAR zK-Hqy7E*!~3+_2$&0XHd)OgI~L@DsSd$0W6vt7U8l#g>XqhWhP5fvm$ij^bp z6rz#4uwM{h|JLYRIh!mT!EUkI?jtFJjTX8JE{?`36C#s!*K+iGw~z%339K3l_R$bx z^byzpW;b8HPku#?Hx&~KPV@2X6Kg*Qe}FC?i(>Nl3)H3zK%`IVf$8{E_1kL}sp2~ee^yXwY0hm8aH zU65$y_?+Gvv(YgNzV5m%a#^kO3?7~l1M#?|soO-w{f;_%_7J1sZaeX{cc!Iwh0T=r{zQ}?^9 z$9qv9ErV>odanTD86k^+j3ev;9o2t@@XIO%BX@`kJ2*fHzI}DZK z2a};_@;9}9y&X>Jo!-y;R8~0EDP7DXq~{|I?vGQ4sAwL141qIUXTzl0!>yRkvBatF zUK)NEG^Wqk2EFu^^UUA7!`~hkWM?EG>#uT7Qa)E@m(fQcPnaD;5D(A4Drq%Fa4>|v z)b5M;qZd~EW5dk$ykk{_vTBDijAMW_xj#b;E1g|{%18DInV59VRL5+|bjY~TH1muO z62`ju5jo#FSX`M;Lt!WhNKHslBPS4Gb?Y_yw7j`GQ8DuUI zEoJ#kca9~ZJ#tUwx>Nfl&R2UWOQXd!(*5z= z+Nb7msm#;f3xHRWN^Vc;r4jRm6w>VmJ080RP?T6q6c>9aDvMhQS_v|`M)s^4xk7l} zRxBAE6SN1`etanjG>*&5*_0`VsAF^V61ZP&MBV#YM#n{nx%)j(b`bS%;T+thRV?*? z9TobDl}urYz5h;7A#!4DJjJbCXrU~Aj}d}P~tC<_T=5>%%h5v4_NwLfO{ zmJTLoLGlTo-7Qh>4tOeK@wWjs+2YY`!3d1Wvj>evjt=PjBhcI$$A(j*S-m95{Xtg$ zWMlDEGa!1;EaQRG{Tov{(kD0}h}TK%1=B?#r)5NuOuck2qk;SB zt!&u{n70^Cl)9Vfp0iY;R7S-v|MphHaG7H#4L&v(3~kQJVt3+vHyPi&EFG#&zX~5p zsK*ohb)_1i3l9NVp-gSjcg$X&K`N!yj7K#48t$uHC0IMQWqOGT$Hp~DXQoXO9co9C z81e07w1g0H9)G;qc}b^)TA4_)YWfa-i&4!{0kRkIGn~Dv^4&*udY<}3YeF;lsA#I4 z{kv$1V}p!YE$<2;LT!k)c71rvlN~mJC=8lgVDK#0P-+hW?C8CwXoY`HS$@1aCG(q>Z0lMm<0yf`oUU8hc=E z9nru!T++X=NLv0`KQTo+FWshvQwXT&ogprq1)>k-zD-l1e-$9S`lS#;C7`+CLt9Yf ziI#lbSb4gXp7ftvnT^g(5PKQ}tDr~u)f%~QC<{}56nL^<%;UlWHBc8oVSdc9kd%|& z<5UoC&#@%br}BnS0aNKQk9ojC{~fW14O|p)?7IU*Hwn9|OG@=G zYE*lhlp=Xa4)wzMA-1glD0+Y{K8>;KL!T8kH6zSBHb#?dh8LW>QCK=Dd?URW=jx23i@@$-=04%Hdp zugjZywqKjH?QLKhQ0mAsz>{K1yUH`f)98-hT1<{Ei!Ul+>4rthm4LnJ*4`HF+NN25 ztPME&2q-YZ^ZDmiKsYejiSp!vAOz7dcV_$t_E{o?B99D4`D-e9v0TDCfNqhL*Va4j zLE|EMo_nc#$BVpr`Z#?3hYhvnPiuW4hZcGqD&1!gDmF{Cvc7GrqAzFrhAk8WrW;uJ z@;LjDq$^4!zUiy198L~SXVG97`A`=6c*Y9$!|H6B^%XQO_ZBGyC*4kcp2eCxAd_WnGmitU2PIl?Q-psRXtR4j2^9A4?QfoT-k7^^T*- zo#}5pJ+`)UU0M%YJw6v*T@Pz^B+K7Afi_tv8#|x-#UBo-1&sPl9p!+6r*G|F0ns?@ zNKLs&Y+e5IT1%uC@!Q46=2ry3s{^>3Ch8QY03k$sUtW;Hty6%d`w$>iw83?T<4@UI&O4c-bIkx7_7g903cZp z25cS#9eD|r+Grs;ChF8~`(8pd-JtSaat$#!O_p^RI+L$9SyD)>Y}=xE;;q?lm^4(8 zyGrU=1myN{G@?jbgc6iFk=mGwIMz>6R3xt3`{5z&B83o8qC?_*rWF_pJ<@>|1R;3o zy8`BOUGT_iv3$YCoDt*TliLgAvS7*iNJJhPGm=PDhDhNgABa}dlYiIN?OU_mhH^v%c)h3(JRg0zTW7pU7w$61-J&+{{}u#a(xf+z%y^lC|`_kBzu>3 zyW5Rj3+CiW?3Qcj5?$ky$8->PrVFaMj+*P%OXEbsv*p=-Q~qRv$RP_jVy zo4lg*pA~~;(%m;B4-1IG`TSM+*UU_RdLEZk1o<#d^Ce_L>YTzzCR_$i2&6tmL0D9j z&*MOVJ(q}e6gK~Yp?if0MRqnSdP-F^l?aqrTp=_~85lpW3A5l}baV(kN^ur6hFC## znlD#9c?9qA7bg!0L9LavUB@yU?+ahlyxV_cn93Pa?N!|dU7odW`r8u^YG~Fr>LMMr zu@OS}bwvlEgZTdBz=x7*F-&|c?t)SMNOA2bvnS}B+b5+qyxl3N16NEUhlB!Q1~^)X zXgzNt!?vw9FW-P9@npRQb|Str8GKg9YaZ>I71lnAzkcI(N*>Jo26k0G2n z>(yqPh5mCw_Cy@bG&`;DPnk7+f1ucyxk}7Z<-KfIuma}YE*JS7EvnW9F1DZhhCM#W zEtybEaeTCLY@;$uT-MOSeXxhD=~&Xmw)t{^?ma46JF!F61@GOt87+mMLeO3KH40f8 zHzYDOOuNAZ<)y4DxJZ8?7Z2Qu;j4G~{`w8`r2xi1AH~VthtjLz_2@M8R@2c~O`{(? zgH*1ST`OL91i|7AJTA>u8604C3l_4DX;CNo=KqFNTa41VMfoO$kv7Ffh1L#YS7^7D zNn@xPf&!xBeBrX-50zbQ2oj#C7m~uvepuEr)$+3QGf&H6P#?FDudQKYx*b_aeC!Um z@Gch`kVy@3G~pX$Am*5+~$Eb%Xb4s$BgNmJsLce4G4@Cm#Z=34d0LK#_`P)25` z$!~zphfLj#A3{$PiktDqQAg1!db4w45qb2J=Ud0w#@0rwpDXX2yCA!F6~6GwwnCfz z$z2tV_l)Z-;AeAxhoTW8@<`y32`8DPyPU|*p~e)gE0T?F-?1p*V6FV;y6=75a{WCB zRj4MKW#m>YJ1_a}>`ilR9an_IUbK$YaeVAR<4Y45jRzN8_xv?11}e>vj=JzRe912h zkDy;hT(j#&&TR?3AF*)mTn%F7E?#>L7q)sc1UC*pbhqcHm*bscx(q9^M#~y$gz0;W zEVKTVuF0TtVCjVr2dBGYX{fCSGpsgfWU0~QOMW{1bNMoI@`GZlYMJjsaAqpw=TS_R zBop&gjZKCbM;?o_1LaA^P}wI!o^d5{3haH}9rX9&qpz56_&Ip+8<#WQw9D?&#uM=( zxUCHC{{+SNlS6t0mGT1ns6JD6_|XWaNvynCzZ(iZP)RzDPr#DNhJOZcOb2I6_JLR1 zpMM=6fiX08Ge#rxe&GpEb>FkHoYE}JS~0d?{g4F*A7OyWBI7=|@*iZ+JaoI)m*E)P z#R|AaPQZ2|l>_e?=qF<993;b5_AY<`kLMV8g1A+}R)FjnXtdvdjB)_!X+P9qx9m7vUQ_&s4W*-UOP*i@RR- zx^UQf970o+Y~T8{Z6!w~!)WTR#HP>&(U}yVxcTGyf~xwLBTA1LAL|7&MjFOOjL7mP zc31AmBWpesB#T`IvNR|0?Cegect&8fg;s>k1u%qm6>L~c`rDAQP|bfvNv zsjz)PW*frBi#;gkAVgKW{Bnx=lvDTw;gUra312i>{ccd4(-O>xG^j}(5G|qhDVRwR zqKCV|m8Rj}1g^gO&H?i~{%=_hDG>zQedqbH_L2^#PG80Ny?jJ)#RFDl-UC_qLFKDV zrNOo=5j*kMXy;-9{Q>Qt&+{os@JJjy)0fURcLYqe`>mwvUh?*QgJ@*e&_9XpqRnv> z1G7&lm=A1ae<_4vAgT}9LUjysOCjNYL}p&C|6!(5Hk56LD9^thCOW=K=sk=U(SppS z`RC9=F^hf-=MKdWQB4DpiS?^`r2M}0e~x_$PLBgV$bf&|y7b>4`xLx{7hDIUxu;`O zzz^{$I6*|ZkfI?4{n*x+s$`?}e-9B*%mOkAL56cR-~<145>0jp1LPX>=0BSMd&mr` zlR!J(stmweK%)TuV6U81VKs1K&p-cr$QCq=>Z_V&rT;$TEV}j>=*3N$bJY9aL-C+t zaOJ1pSN_i>FyMnm%f9AN{Xd7Q*+9b>ZkiYN_^&gHNE=guMk^=PXuj;%qGLQL9n@LV#Rg57@w__OxkK;u3Wyr_0r1_buV zZ7E!HKXMGEP?=I4fZi!Skw*oA@9prExQ72+-Eg_RQ;H~*zP>%*SvMq0>}}JiVdQ&{ zBW(k@VQ-wfY{gfdmDC;a(<~szl)#!zTFToC&(Q-$;Qwd~r#@ydYu( zUri1%--@HP&kCr_eM)3tvD;2^QO@m8yX z1>l?N3P;4HGX6Yf+~pV>J)Wbij0~OK#*&ewOSv>F-v1;<`y_eJ-2~j*m2rw$J4n4J z=my>IMzV^gKl`!k(ArAd#*e*oXsD}YohD>ga3F0PKN&Zx5>Yasin1eo9z=4pvP7Sm zh<9-zK`2SRJ4rdhKp|(_q2X5}EnIR(l5EcqvE`SHb1QrTJ($a&XQ_4i=I5#9H%7cR zF)N|Z(O*y|_NP z&(#L!wxEh1K<-8m`+?%^q@Id3GBsF17L8Vzrepe;1@vqAT?ceGG83DYB?@6XAbN$NaCdA)sH#q3@_8XYbYpPO{mCskOnJ! z84RN03V_7hIlx0z1q*CmB@_UX46_|q8F8K0qI23%f1=}+K4VK2A1z;(x<8odG;2Vn5h zNHUrba_A-@GzEUw=Z;^yK;7ej75p%$z7^QUyaE27ArKBY0?>mXpbvgF#*}q$jCWz` z0+?-D58-ZeLu#M?*4K4R>=HWzwi5l5z~faAOd<5Jz94?d%Mg(i)itP|n&c*H zr1w8Lz((tAd(AtoIWlIg1af=+qydVA!bsaAhO8{i_*S4s>q_E{c0wc z>d^zHR@NKPPw}MqTF`<-@5YwDk=uwgECiDku+k;YKLLqEH0XlxWyc_s?M1URk;|}T zI>%WM@H}L)6f77O4x2~iXLi(K*s)9^Dn=0SudU;Q#N&ONIUfm7um}5KN$huafH=bk zApF7%M4BV7#|lC-Q5Xc6825$T&BZhg)}4{gIqk5-z02kW#TUT(@t<;0I<&DY?)~uC zGKdLcb?rVA=4nd(eQ(n?>a%7zYsm#UqH)a-5vmj)c9Gi%@(5?Vt#P?tWN=gt2sAOU zF=gItnVL%?(y$e##yHO{;60h-1}5=I?)-Re3n+RA^%OA5K;Fd^${!xvD`tuQD?nE$ z;r3-9%>#OFv&<4I7w-TzbjkMB>hp_^9lVq8pFc9f`(&ue@-=xEfNE%vkyQ}zo zlpg~;D*%P?0x&(?XUi)9npb`#`>hucI$nTYs;_`Eq~3eV&W8=8rNGR{Yv0d)!LIiT z*LVh8?PK$=bzT3SflRq%-EMyKh{LImXDS`ZL;Hvh9;imy1K9S6vPxtjd9!aucqPCR zQfuun_=B7)$#D%TP@EE~6wg4#$|*>@!cm6TWv(z-)Bt3_FGROl&3Azmt=$DcLBOp6 z-O&<3z|{WU7(lBzxS`{?HNH7#8hCmG79}fnqFZ}^5V-@SZIOXXdhi8W2Rhb5q+~5Jg3}^*9W=@M}5tVucHX0f6Rt z0W~wcU-=XD_4R6Gf@gDC{=RU(TyNu4(|&cfoRpQa+-Jy9=A;2g>=8m(FzwU!nw(=(1nZB}hqfs&PS4AUl>`Zdil% za>NiBz0mPFCIU3$fb2I*v>*W z$NS5lVzMC6{R8!hLU?QfP+*Y$${9-f13>W<8h6u^Zi-zy&mEjz0DyweMd10kB>bTd zIL(C7yQ_eZS@jDr2OA%vXZda)FQc#gMIWs-Vs1;`d>0)ve9kT12p9vSvlouv9$~)5%}co06GZ^u01d52Y_paG4E}Gx$_d81xAz>9}DOg==b* zI;3PZbQ0ZAD)1d(GF3D(Ad?T&E-U-SS71@cktp(u&bj)e^@)OX%2~|ws+pc}1DK98 zQEQMu?{Oe7Q|;?)iC6jU#kX5Y1j6id;rFX`?UM=V>18z6eC+Y2@ljGj=(tVy^XhUS zkhv*_u;pkyKJs4o9Jo}YDDZN24xk7;?)D16MXp~q+d}u#608BTKQIl+h+g(8onqG5biG!g`=nZLl7mIRbX8bOpe(F=fu#R zMT%oEfeMlx?J;4Kxb>?TB)BF z6rG^8E=OYTy56KCu&hS|&+a`pG`;lH!AZvcp;G7ywhK&|Ez(j9!^=;#L|!$5J7JjJ z^M{{3#>Qg7=qTyvD`hTRQiD_}OqVG3>95ThDQCs(aN4L|DlW-*GwOSCNoQi*CGkjU zqALlt%dI&{vj%VZD960}2U&4jD=@#8eUGN(|6Y>LTWTdAyh}%dj%kiWb(cYy(hTb( zsqvUhMqOXvb6R=nVlTXnO3r@R-)SrI#ZoQY0=3^0dqwaH;5ybrKAYf-_3jw@Ni~%n zp;DVSn53YprDvRkp?UCJC{?$*}S!se9;q}5ihu{Yx#i2GD10XlFIGO3yus6jv#HQ zg+mEVFpnfkjj2nLCzvQj#XCr=I-&qEd=x59uhqSa{fV4NN*Jn^QtUe)33wQzH7z~* zY&mpBsrX@g8lybJH5L~HH|zK@A0Ptzpg(BiHt|4_}`-} zo-DR385#$@tXKBAg{XZ}rhH10Gu#;iHZ*u4Qf5)m9%c!ln03gK#-d#AjN0%|*FrXq;6(^3vg~gpwie1x`IcukP zFzoZuxF0$i$*A#oEF3%fvK4|$&EsIZNt6NR)sHP-#Q!vT8G zOdM!%KcB}Q0yfVCpbT|U_y%Z0;3K+E9`MPsCl@2WbRl$?>_kt&+M)m)hmD9|En2Ut zv^6|C{siQ6$CJ^k-UQsXCxs;k1%JP=f^yO-kI1)}- zQHe}Mh==2d5bAJNeOt!>ekcdXPrFtQLVE%Nu6Tag5OWM;V6R{@t79MN~CvPb=XON9mk za1siypITRx1n^#BJU{X;c#!Fyc`<+Z^b>^Q!mhen<3fK&bIB?Q__7lOiDJxV&C3=U z^$~?7o(*T-W9Z6Fo#!5^OeF!A2x z2p?q$;RvDq!mUF)g^u#$&Tk$V#D$|WZ>(kUQqUSpi;14ENKnTE1i=ak`f0^j;-CDc z4&&%#-rfm#d$8YD~r@`@2Cyo$4vp>-2!0T-jk32 zboe>;(q}UQ@)ru_TP(~+U$HEV(R6<9zMr@GO`#IM|Mm8Z2n*tNY?CoTOIdFwZoo}1 z|F13UAAQR0w&Mq$QxNTQlOg>5d@+!%(X{N7A!Z7fvvvf85~G7-)+Z%Vz%l_?HVZ-! zZt71HwSfyj08fRFOn>3=PopswIvfsJ`61`sXqs^;c6ZlPVDY6FO{r8Sonha^NJ)2wFU({>VE==pP9`QGL-&~4k(z`d5JYdkt+xACI4w8^bgIxk8CO3XN=NU=~P2NYSe z1cl2i@g^hIO5xOp>C@y45L4ygDM}HhVjX~XHK&@y1k}W)W6W@!&*7(TW2d|8m`bLc z{$)Me$MU>+y<)`UnQc^|v}4y*ymL$@5XsRZ-RuNxlV0fqa3%1AZ6Wb1QLK$A5?H#< z{qXFPC0twENT0-07cm!ht%lG(clI#~Q!~|O;2)zHl&#bPUsQV>IS}fyz*>p;#ttFB z{wDn+p_#3~?$FgcBE{e|N#q9#1m?Fv%o%q6l>2@ndXyVYtP<7M{zcR$#1pvVC!*IJ zMdWZLFxYRa;d~Hyc~SUPa2ww1W{=v<2upk3-8Ab`jE$BSF{f1-h{-Eg<^?MxSWEhGOE)ccFVdQ&MBb@0ODv$X#fBK literal 0 HcmV?d00001 diff --git a/docs/images/design2.png b/docs/images/design2.png new file mode 100644 index 0000000000000000000000000000000000000000..92d3980a97ab18a98a7378d5a50fd67395b90d3b GIT binary patch literal 23696 zcmce;bwgC&_x=q?3_WyrBP}gCbT`u7ARtmocQ?}A-7O8$B@NON(%tntzCV8Nhj9O) zFtBHzIlInUuj^WbDJx2$A`v1%K|!I)NQJfd2_<7>I!{u$H0sP*4xRk9_|?LAkI&K^+)DLGhnXX0NGS}XME3 zKe}A6&H1NAeNb?)j!>(AhFmhsrz;vx&M}KoQ-x^E__#O>++c9JIRyg)!~Xt03LQCl zcN!QhPEbAcljryJ52jc#h?q57#X-V`G>b)ZMVXKD)2Ha2h@^Kq{p`xnRWUCp&|n~#Zw zAFmp&YE30g4IXd0eqB%~qFk{G#|Wlpc;0s0n%YzjBcdT?aopdwv~;+wzo&QTY-h`Z zV}kgL?T)?JjtZwgTz!Ab-NZJ(?hj5Qx11ld{?%l^zwnxs0tsRoZC&}!{Eo`>C?-Du zCg6nFF$T`dmxqDj(-C_9+g0V(&&ekYr3_0rFiqdxPBVmDoFX(U4;LKlMEwsi_};tO zKiWRbYE~}KSFTT`5!Yl2eY~j3d=J0!+0fb9JA#Lw+HCWQk7&O7d&ffNkMLw^C8bdo z+JT2B8>5~zx2s4HI?t}04tNLyHV{^g>uN`NbtA+k`hF zuNsoGeYpAg%PeoJPMSFu8g}lj79TC+J67m?97A zAlFcuy<`v|bm!&VA;1|=8^1M|#UQZjRytIe(Bhd*o~*a_z2exLAY z$%pD_b3V}h?_0wU6KXdfG5PQ3x&Bm{%8K_O4{ac$KE}f%&-l`uZxqD3&F6cQb|X za#HtWQAmT3Mjjl-dS~K^iW61Vt*x!K!5&af?ppI#l=N(Dy=5FN%F;$3C+lFjaG#c$ z_(VmshK$KU#>!bn<$>VoEAdc5bt9<5O#5p367Gv*Xc#;ey_`n6rrv-Fubk31l+*_7 zPTNC*9ydl(p{>TdNO&wOg>uOUbLAc!Ib_~)F3w(~q%1MHpiXB^xsd^;ObcZZ{7ulWqq=b}|6#e0< zLx*2Y$LE7&w{2PrnX;w-ooNf`Fl_5oF#;EJ9}Igu1K?4=p7kJs|<_Wi8o<52}oR zk5`c@aign0Ki__+db;1seyd*MJ4ZvRTy4-{@*}+CX4$qV~?4|=7CeegkNO>RE+V9i2 zY;9&sa!-(=kI8(VYOQBzO_pF)aIoIhw0_=ObL>OC0@bU|R_VWYU*7Z$`!lWo{BW%z zc2mF?D9r%}ElR#X<@oaSg^AgKyj*darn_wY>tUrM{C2MV$^J|+onnTx77IUr8co?i z6j1<55*87fZCy9)AXxc$rS)BrfnTuok75DWBYZTJf$S2B`w%peUZHi*%W*<2>Y6mo zIXKvaFwic^8aIM6SLZP4YfTwcjZlC}_A+tCli}kAh-j zBbRv|og+NAL8A3WmIJpn+f_F3aG^S+B+*&xY*P5u^ZW-MU!&yF`A{6y7zn!~x4(W; z2w7e|nCjDGk$kHE?Km}zve3)jZ(aM=p!0n2Z|Z(ot|BPA*w*8wU)`Yl2G!!?uKQi0 z11KIIUr%2PV@##Qdl=+!-cs^Vm?Qh>;?$cZaz7XC#>bhOc%;!^L>VQi(oU8mydjYMX_fgO1kPP8|M;7NYV5voV73suL zN^D5;h8Z!Sq2m$>M|yr7i0Yv4Ln8ri&(5c_IMSP%nrbKnc9R$Qygs{miet)>W23o& zk%K^+c+t#>3~G{y>Zl!$$0;D=xW|+4=*Kc_C{RI#-;`wi!StX!;xUBb{3;WY&LxvF zEoEyNG_r&N#=*qNF&ImNSJe-aXwzEe)FD+|j(P{AF6J_S155`BFeV{~$3zC%VB|S- zW3}!_V_GVVUYsUj8WI~n2~E3g++22WL3wj-b$C3)f_Zu^yv)!b!x$^q4&6)fb??j$js*=~ z?+d@)$1s{xp?JJs>c~T>vp10+OJNTXjI8c_U?in`Gz8hz#SeZ#5y1W3`f=sGVk}Wq zyGkfSWgFE)+o#`>H;D&p>@VP^qUFCnX*zKnl z8f9J^EB4?BZI4p9DAY?TBVKzY{YL|iPhmb?&c>!erhL@z4Z<3*AM8@MorzKT9-*Z~ z5M~R)Fbj40R}3yf%fq0#wc8hNA};awF@eL z9A)jQxkDIczcyV;P<^?cQxBric0H_vl+V9fO(a+?#(yZf)^`IN0Q}4MS$)91PfhpzF_Dx&Ih}QmJqtR6Tmt&%(ElpL`Wouocif2 zI+F1MJ<&DdXky3@JZni1 zGogjx!Sg!0xN?U~ng19tW zf>h2DYi-$yN~o!4MGptEueD=0Gz=$n&>Q+95&CFmEZAt(DO?0S3FXpZOe%X`0a>9J zRz-e9SniL@i9fVfL~R&1%hcRwTOLB?CJH2R_?~ugJAvp9NjwCnJ!PMX>8K(iHnt*` zgUk|VW$Xi{J8pgYHHoHL7ecI?)`3OQbiWwIRbk>VOBUKU@l>WzK`3ohsnYG5h_@Du zh((6|IJ!$8lraT-|pCYSSU9-4oT)~e^ z#IH^gN4`D4j+pm#QCDb0^OAM&r=zI|&gfVvw9dI<<-{r#I6YFt++sgMLMnscN%4r! ziMKUIB<&2${yQe^ozFpWt}{7qnRl_-G_qMywDHvYc#XK>m-677^*jBWRQp}tCg1R3 zl5mlCvc`mklrGNZRE}lLY_-LbbL!x7PkZ=e#&zirM$qp$@TPiQyuK-~tG}DjjFey; zpER&FH$3=3YysuNs@oHRxAM)pzOb+mQ<#uCU+M=AsVRa*Aqo=rh6qB7Yu0f@&-Kdr%_(Z8u?Ix4C+&j~x9lev) zq_{ZO%`J&FjxMWD^G6)t$ugkS2>}Y}uQIYi4qwP&|C4iBNAyX1cu&c*lb;1)nVYi^SUnliPHy=r=M-j>-T_hVMbZT|l zc?0vNVt7;V2neIe!TMF6K{@1T0XmyCzP{c0Qd^T5GK^|PdeZaxp%ox#by9v_QKbqTWMlvkhLB1ly99pcz^Q1*vr zn%hM|sF^|T(Et?N!w-l3?;M4NCX?*mvRep|Vk-MT^aTSCTo%$tzU2Np!;#N>%@&7Q|O9;^6>*{BM+JC$Z8c6Nm;KyVCul1QQ(A)EA)@Ue@ z7X3ZCQ2^9N*d1H&uk|Yhx%eNSTRlmp|3Nz=5s}a-s9@@Ut)I|=-d?}&ahUx3yl(*D z1EHY1Fyy_Jy)_a$i_m6a1c#FsWXtNkz@O zmm5$`R<4eU9#iwwskwc$-gT-DfqW~Q>+KcrJhh#V_|ojK+6z^#UU>kY zvCG))!LT#kRH=)EO|RUf2ZvleS9n-Z(C_7On@F!x=NOlVOHH3yw@sb3*6iRA7GDT3 z2NMOf#S4}{K;V73jyWP1!~DEJu|3uEhE9>V+F9(|(2(>fkB;g16u@-A#MHFXVw7RT zVj?fdm114?zl+AxYn%K|ba8cknFYB%_80f{_V$u`UWJXJM2Lf8yf1e`-^s|t2%XD2 zT7`1)QS^xZ9(Z!RJM$Ao7fNE~4_~O#>4xch7(DGdUhFJdd?CI+E> z;r70*B5_-PSK|1S5;!kO{ZV`GJny7>{6quDyrz3|d~at<15djMG@3kvguD>+WMzh{ zz%{FN*V)zybX*QgMG_J7Q#P(NB@aVO?bb7q)(N_eou$^7N%Pyj$E?2x2aZ=ipLX}K zv%^)c{;8RovQ~RpY_}H6?sWUcsK(7KIKCXJUM`|5qNPReP=ZH!1uJQN$ov-vQl)04 z#bssvEYi3OtBVtvP&O12d=XlelnNCO5?OqE>)GVg@8jdf{-n$_2Q-~wEODJ zN41mkoqu;m-d*+i8vPnSUdC^9d578_fa;)EfTKSQ$6(}3k;#4C zN8RQca?#T83;}{t)4|9pt=e{gnx#H3%r+t-GN*AdtK_wC+1&f@=8!U}|b+f2aXC*5%9^F)}fP!Z(d=+$Rv5=bT3@9F8mz}RfGS!lq{ zZ(I4sthCmSF;og*0cougE8sw&sZq4~DSfFNZWrF!L>sJ4*VoVztF zyyRF%9YY|M7}AC%#xhn_o&FJlH&=kJONN0F$Kzmtj-#NcIN*qd$vK?MilQkk9m=r& zfrTC1(-Up%%o~`Ph(e4O-j0cl1VTB}Lu+2dqTu{MbuAt`z=*6Avk)~sJ$-k&lTfLf z??Rr7fMceSI?6*J!##?bC?t#3DVUfKzk}HH8a;%}6UtXWnCWRq1Ur%W-fg4X4`9M3 zV-78w!M#DqhUwm3g?I?K0)m2nF@nISE%8pzdo9I{_?`xR%DNiEeqHn-R@T~g{(5d}gnMM=!`*P3SvDDMfNBi58)hkti z#e6^QOjqG8&p^H)DB8p2Bwtw?-{U72n(Q& zi>(OO&Z}uTlJ~A=xp-NN4c0TCCk3UzXmr$WC|DuKQ9NslEuW}vvhTJc3ErKxct7kH zZ~ch)&L@H(UZMjjGapT&LB-Ro(89Pg__Bi&s=%jZHzO@Aor8_1tMqn*6sLpl%Z1s? z(|v0{fhCL+!1bo4rf!aZeI|;9&y80SPwd!Vi6j;LivK$ghUJsQ%(V0Q1?mpG%r@$^7^H*gyw?DLgL6c@R(uodK8z z#?}y{eY)w#Qu4f6vUFSfQjC^@@%~5L-~4%wEmzC*h*@GNEIN8#LUy85t=KVv?V3OP zq2keKPXmqy+ElTqZqpIEU$oe>MC@~gp!Li&(HtfgRxan2O5WW-fIkC0zt2m190|Su zvQO7jsd6ri7nj{)jmPbYnZU!J{I=G^KsCyORGpEn^pjl8l8BzzE`XZ##zaE9QfrND z=jk-sY{!ra5_Wv*HQEK{DN9aZ6yTzX4n?Fbz8YofqCXZ4Jff!P*H_ZB1U=ZUIP zQONNgx2tZSB=?bp?3Ww%fe!~=46%TUpht@=*>Z3X< z&z>_QchGt}@nvHj9=9BpJ(*ndaScEOw9N~1^u32r=;DkD&@w*T-n&P4Urq`;I2^v* z?~&nV9W@Ua4WAQ`qdYusCny<-jxiV@)bh`Fk<&3T^MI&OQjJ5j2LSj^qz|y@@0yi5 z%2Wy)kRdw&%`blB-Vg!vGKTBTIp%2L9^Y9Eyx}@ptbL=hGxCipVZ#LYvI*0k58zz# zGVOW`>2xkz+)gZ#O}*MA#LXV27{MESy>}irb^=`n9bbFKXUUdd4lf4Dh* z1Z0XOtlXc5@SsIPL@t|IM|N)pOjuYy+Zd?tl6>*PUoN)mEfcD}PFw4wk_L~$z@MWO zxZ@K@zjJIz#gIM&>=xenM#ePSi%T5YNJqA16hU)> zDujcyA)AL2oix3&7{OkLq>I5CIQ!l8*rwt)KL80;NMQQjm_yYNenwi9JI8HTEmftl zfYxoY>v4m}XlI#2xC~8V*6lqwh;Qf|Z(_Qp3-Lf)?nf*Ib%;+&D0F@sCU8wV_-&X3X+viT zJrGWy9rWm#8IX+-L61v~#bP&Gf^j6Vjm`rv1^%EUfnZF#<#}|LLl~z)oCPqRy32qm z@?4d%x+H6PqhI&r;HCiOM6QAOx0liKj0rM$Y&7Z!^~C(np?BGs{#4RN=nG(zfUph4 z296P)A_@%dA(4YvY*GEk#&@RXYRl{xs{2Lh_PxSXJs{DL7#qZO%sg?IlDJxE*iO6y z)e$54Us_Z0jcr>zT&>1w;u>BoOC(~1B-o;5o|ZXkvU>Q!B(K+f=)&74@yZI@x__}R z9V`bQ%;*GEmWQLD^-f^zC(9p_h=Jr}=o4(DD>pDt zUPv2jwc5e@%>C3+`Bc646Yo!J7R9$?6kcE#IptKcNz&Qq0}D{U*D~(|QdknBMtBJW zs`bQAIQPxt=JkQd`cLW~B(cdX3yd()%(wb_Mlz_#cZ-8+>W0E;$D)YUu=Ej@y)TC8 zthr43K`WYqxcZynh=VUo;~8CWqD{IU}s2a{1iq&B@j45Gpq-=um&2xnp_oh4XbI2IiEY(t?DMHi3k zhJV@$$^*v)N%xPI+@ab2x5)!1D;O!O4^`%0KcpRY>Lq`b!>S%S4+k&&?l38x?)j|H zPO}$|^j*8>5~Fn~PW=)I#twETkVp!Ox3Ieks|9RBZC1AMOH1Xqqo4t7tzZ@m{JEoY zdeu_Huq)WKK}px$-FbT`ZqbHVw1WSWN+TcHrrl#;ha*3qLXW*F_iquT3T2a81`Vn$ zLy&OHMp;{+?3wbn$v1f9(B|aal!dVO8|nwY-}lIdiyj&K@qYRemmg8{#M}6A5snC8 z*lKYGg@|$;FcOBh-E;xsTn=Xo;g*IOT4<3Yn>R~63Juc8Y~fLwIbO+y@sd^9Vl>MJ zq(C$lvO9z(+lcesqO3d^LbV-Bw91xpWV4q(#XurGj6Z13b4-E1T*|0>$RwQGz zecu_)iC}&k*~67!6@i(^Mua{)6rbP{Q(+A;UHjtnH^hw5Yz}4P`-D9=+wqsjwK9xo^ksE$jkwH55!sRs2=3xi*EHVO*fm6loyzg$IdD4lR}&LP&o5evFz&}@DB?Tw0x z4y4)NvGClW4sS*se9qK{hZ-(S4D1i*KfYH!jbTt*_;^Pr_}jLCC$An7mr0MEDue2gs$&XRvQTHdsBv`qR$Iq=6@_s3R1qSxTqo$vP{%#*(*{+|a zrL-M9=wF=s)%X>=_OO0<*m%=;QP$Q;Ip^!Ojf%<_DoQGUF@N6sg9)&_3&bD7xUS4V zCCJ6kC=7v_&b061;NsK|hJ~v~eA}Ir%4(QM=ajgZ(=8%`hP0h*yMxVeCb<(~mal)0 zghP-6mw?(-D`66++@`)iiYedca*N3qWEw$U>>d`MwByVCsdy^Z2!3Zcini@DV7J

zLhH8mx__UmE6(55oq#4a$jyZ(xuVy<@E?NCT0dK|&$q87cjsNEO~~3oA-Ct|8dKy_ zBvI2356PHQy?^&$)XS#H35_20(SY_S$>FLn0uND7N?hE4mJK-tzjoxIIqHrF$&x9y zf!y&J15DX|lOh-M4^b{)z~H{oUib+T$N9$vMZ4bs&`1zUDGgB6|1%E?0cfSu2i+X@ z-+M>_%J~P{eOmn=1-%9Oyn6Par~2zw0D3ut0>C7CcS6y>_MI^R20dLDBk=nNHW2_a z`4uXQ6X#!hFW`b+ucRZO|K1LU75ED}-<%cv&ywTsdrAdd@bOkz^8Y-E(0^f%FP*|a z66#3?TyVWBgahW^J)r;PvQPrn|Ji5&$@B3q3Vjhw#Q0};bocdb&t2np2U!0aUb8Sc z+tl+y;j&%R)zy{oYYILw@$~gPY0`Q)u@Q0#_wnTA-h}JvQaYZOZUPoQzJSBT*xj9& zp#32Z0Y++Esxfp#PR>T559@M=t>s?Ts;LQ*g;{|v7=pFUq}^_Gnzh;;FYFTuY*~+H z_9S6C1rQ3#!F)&YUwa_m{n)lJZ4!fSlS}$dJ0Z_^NcT;*UkCEur0rDBr|YyvCTT!P zp<}V1+zPrR;Q5inf!At+M}_t%C=5`|1yb1zs__pIylN9k_DWZYQK~D z`>N9kkc`SdXjW^patR7jjU%OVw8h450MMn%u(?R{7(Q9HKs)I(0sR_lwvgq?N=|pq zfP!vH&gVT$ccm-=cH8-iv__i`)hcu}G{3|_O~2fZAP`3@@y$FMRaKVOnQE_0CcaAObl_Ylmesr9`;dZY?GJUjv_Ble0|n1WB1jrG)~>6fzMn zaj99Q1!Vn$vN&%62x$*7Q1`y_l+rypRN_I`L)aOU^_WOfTbJ^qrFDWJD5poJ2 zb?Vy^(P#dS(?9m2{?iD2U#K9GK9?H#!PiWqqC*+lGYeJPtNFGI9OkmRYQaVWMX2St zW7)sbU#nQ(Z3{Ah3~r|!S#=sp95;J4 z$~C#7`KYJ@I&A4t)M5ki4PXab)q;oOfYZtM0vPsgkeR&CwFzX%lkV2C3x{a@hc0x;MzF zqp^Hm4_7?_2H7qTNXZIIy>NEMBN>*0(>rw6C87f;V4x>5%^t!WNZ+f)#a^Zbd%vE{Jpf!kIej3%Lg6nFtp6u_iThD8x+fX6TWo#ljOwii4G zV2>glK5f68;92+lofn3U`1jP0jlk$G4~vNCL&0S=$N&-C5K9IU8h+XN1^^}&H}`wM z5%~a67kukXewUQv=ZC{;*7mbrfNRdy`M>8`U@!MWJA}T!e0^#cWCI}lY&Gz9l?~lRjX}SGSFbkE zmk^f`zS{`6Dw*ewz18NC*nS#j4C;$^vILea8;*;VG^(QaZTcXnvMs)bRz5wA%qx~vyUeUnfeLh)fy))nh zOyHGuwa!wdXuZZB%^!aOsEG@OL-RS9$X=_~1ddOcBO2_?=CIbT|J~L2*RN)W>Pk*O zL3}fS=I}~il_YHsJpn5TME~-9yN<_daJ14|uPWx1Pw_&QmA-5V7>ay?BPSgFYSSVb zOeqWetc#$`%rFgUwH^kD9J;?avW|2BXxVWapmAdRF~Tp>iMUKqw}9EzV!InQKL|d`}P*(=F8P$(qM$2jEDJlQ&BG9{Q-2 zCm!|&0S$o04;{B_wm{|}F)*3#&^hc>WFmTj!~WJPat=^TfiaDS!lf1Dis}IIRmoJT zKp_nMQDvAxSn5@hLOI7pQU#;LtF|GaO;V_o{ z1l-WSzw)zn*Q02*d(|;M6I7ekt0|5^?0tN_#!u$2;j+w{J zXom)2Zb(5NosjbnfcChetpG=EiuSSHKBd(gNG~^xkOGk7c1xmioPPwIqyzKe4qeYl zBQ0;6SDcX15GBApByKBy)-;v?Fs>L7&2}eo=K|)DnrsgN*`cAi2171h%Z}p{>`z$! z{`B7Tqp93J1)kMFynX^YdS6GXM?Lc2aa2;hRCd=QmT7&o1r%^Jej{4m?IeQWZ$?bi z>Md~rKLMe`9iS$FH`u0ZcWas*WBRQ~O?Co16ttggFyQ5Fj-RYB20Y;5eNb^o;!H9k zfoBBKB%kCOKcwr+G)TDYD9mVj#lq{KD_{f`bEsgr26Bxivnrt_hfp zf7As~Hi6QbAQNQvrAcirC$-ZNaU!X}%l=Lz(`dzauiaHAtKXAB#Qph~+O@sQSarb- zIU=SNIQNShHgzB=v34qd$9_faGylE@SW^hW#|lC3IM<&;;@HtqRbjN!ntNntFiWqc z;Z~u^GD71Na{jexaL&4!zIa(*b_up?7;pdN$3P^R1P@df5C_Gji3EgOk^@Uk@zaN> z#ZeXf$?cgO>2*}SvkqS}zD*;KS?2Ks7xZ=}GCC-6@st*DXrJ!n$OG2n*!*3RF>}X; zus9?P{s8?8IKK>tujR~{*g)`hBFxJaD=EjRW5VR#3+IZVRi6w*U* zJ}Njr1Kn8@!@k2|9B7EF#==HUsM42bxkuUP$Cgmry7t4Z(m{$HvPD#`**Q(!wFZ&Cy`~n7?ix_$!ByykbEbsh@ z)r|&e>DWu&L~VBOkMye;>*bd&NZcS+@IP!snl!UN9*ET>*F&Xz`oVPbMQM^Bu(pb{t2(jNCmt3b^RnKQ0&4jN4U_%pI;2`fxQo2HxIz%xF$ky^8;rkSZGt`l%-&o+r;;W?Gs zalvPs?r^HGfLCN-F{ID?7GH;1)!}#&i7ci&JRxB*e)1azQTCR zX*0xs;dhL16)mU@XHa?Lb0ZfBUPFYOtSf3&)_r|8@E#0-@mu}whVcH$xde^={9{ij z@yK$a~R*6$@^M< zi2mxQt{f|+WOf}+YgOg#L>etk;R|k14L8S2c$IB=O4P~68LA=){Vl; z%|Hm<19Wobr-FyT?Agf#{xdAS(83=BkdP6$JPucb9nqDTgG7g$)|&T;<0)tUs^9o|A!pI`IOZw%UNuiBe&-PRKM9rDS*BEIoHpAx z>(a#^B-qO6zG`5o1O?S1@{O@33g(5eK!kEib_Y=9Hii)HtW1c5t)G;OC_gYO(^x?i zsNWTnALkykA-cJTm3MmWu{gVG9gxo?(J^x|b8$Uy{g4A-S;N&gDjo*GH|u?_3C<5> zFXwhAT|X|@H7^zjVesVz*iyn?%~Hm!bTEc|-Uky-N~J2WXt2(5rfTO0(VFR8Y z7;Tpimqc%;il@>K48ep-zz9*kHhU&=See_H1VYgLz;6}g-*C=5C%MW8PS_lJa*#0ZV-=n?;boxKE2x+tK_z2lf+Ei4$z5R7SNZ@u3P77ftyEV4)_ zmT$I*{}b_{L&F<^{cpW{M_<4at|6BSyvi#hUy6e7P9QYIgi} zvudrmu1tnWT=)6IJm9LX-@|*`xmuSo%F{b=dgg3mqHJ_VrdFgsF}#L%mc{$UpBuR)9mI>^X?|65tP1l%>;Hj<sRy>vQR8`E4!LSHGfGV&Jc`r$;ly z^6YH=ct9nBgNFq?pp(QzSmP152Rv5Tn1pGm#y?otyO>lsgIi=Pm#--=C3HL<`KY#z z!+(Th3w+sfKW@9Gq^ulm^<+&+F&hx~5toOGUF~|`Mtf$^?wIR*-pdY$tXZy^S!vof zG!DBsUt!Rw^2|@X%Aj3_q6@uZXH^!QK{&_A^HIIpZs~$X?pqx#f?Y)auJE^5G$CH-9yAT`zQRr`zYmGO6J z#;fZeiXo%)v@(O-v4{2`1{mG^8Btm`Wr;k1i@mw8Cbm!v^klO3Cyx3UiqH%aes62L zzC$#=J$Vs&{>eT7PP{Xd>a4ss^}2x7Gu zBh3F1MMU7WVViDV`9Ea2@RxD&7f>qyW1h)>31i`RnznzC^#6|zVnf!xlBiLg1 zI8yM2x)tA@hVX>#j;1$jL|8xqUmlgnsqg?2+a*EH%vI@_yQrAC{KyQ0;yKcgx;L5a zh5&sIS{0Q!_>Y}#n1WzK;PIGM`v((m%Gyre_d@~=;0yx)qXv#%=LSyVG%u6rhsx&(HizpiB~|Qk_W_ZeDf$)$RA@^G<>^78aINB>ty$uUi~egB7oW zAkgZd?B5DC;0{bdKnacMP%H(IpsKxr7)A>A)b; zh|;O5saf|D1!$140AyBXaegt-Sx1L(99MVoOx&a?j4UIm+a43sOb zcln5jhwo0zXy>JP(G%H)@jV*3Pm;DAw8Ta2a!N_TA*sxoZ=G|co> z!0m)su*qgFoHFAB3otAC^u!L;PCFxHo@ah#c29u+8T{tUW)L2@D%gU_B@>s@IVvXc zf76u!0)w6bJ|mmNq+OxKJ_D7PpPySbHKlx!+W>gBddX;H6L}pspgiA9*7}F$-v9v_ zDWB7JZU|{_%nCpPY^ysofr?vz)&V2>1#m^yV3;+TkK$$k0^I0AmA(gXR|76mfK*Vo z4HO^dP(clzI&`I|tS(Nbpw z_C*3Fv7k#SA+t*W&vXIKwlgV!&!ux(!ytMAMl*M#M9jT@igJK#qymCq(?_`=piYa@ z_;|TdA_6z4e5gk*fU8tV$U9{iM+}B9w72*OaH|130s}5ph0LKE$XWDU|4jMufBN*v zGlrNio-{7Z?7{1L?gO2CiUswE!EJx~(YO*NZ86;^9HxTU9R6Fz=`RD}hss{tw5 z{{`bRugvw2Bd@HFOXzRNqTk>EOP%gXnjv=rs1mM6Yxa5d-b$?@smKpMI{<9r zD0C{704)S)7;hCRv81j^F8EuAqf?HEP%Kub>3AAf*hqYs84D4oRgRu2hsGJ!=3S;| zeUFrk7#0Y#X+T#(ske9lzxpIj6DXwZZPX~dD&e2TsY(H~GfHt)(f^K?bulP}a`I16He0-ih{UV687mpf?l%r|d`YyPz6M@VVjOpL}*U+LQbiWM{ z@DTUeHAM|92tcWwhW=kUSN;uU{O=iqu@44W${72;Z>6y>*$LT~kSrl7`)=$+$S8aE zsIe4DSsP1{HInQ}lC^A2d_UB^_uO;;g!}W%bIzQ3p6C5}z22`yYxiFE1Wb7E+bH?O z<-KPB%E>-_W1Xu zaUH~QIrhPcVY~{K%{K$ByP1kP%@2W@3FqBI0p$>J>%yOPc}Wv~hKZ6{#1d>%&2uf9 z6I;fC?U1r!>Q=!MNBud7nqAF>de^KnRdX3&#Iv9ozh>s_>ddB{Ij<7mpI8bDNXLo( z4(21E-&l@nS|<#VDwy(|o*{JHkyM&P_ZD~Z5>}?332VGrm<1_)5E$qd4AD(AT41X; z$^291wK(1A&iGR0B#enFF{Yv|yt&zo?VksF0E@o*nM_w@=z^8d>gr@2w~dd2xCw}v zFX8;jC#w0H{b6Gx>0Yg+SsuoGS$OY!FuJn_E_hjjAYp2`@!3IM1DSDmMV{-Jx0Sal z;|gTBiu&(xet-5 zH*trhT)4%ayO|D8Y!!DSIurJjyY1YgljHWP9GC8)ZYR>#)z$e5jlIc;U`BpC?#ndU zvKVdv-3qIXsCbp1ci36>AOls2zFIBei`N3{Ou7z(VS{W_9DX$oQ}@LG9?-t|+>yUH zCAYt$G%4v;a&8~x(~U)w#6OFFf;@v3atKq;Gec%keo-;>r8qFdn@E4?9*u5HR}nN& zpzo09P-o1(gbqKa&E2bwFe~LPV({0wHU8)i?=SN<9TnEY1qKT1`l`fxi<=KKO60=A z+D= z@c--c8RHF>CM&YPnW76Y(Ci~9CYHuBlp|^*Cm;|4tpl0}P|UJ~uNbx&prxg6fqivV zRTYr4dY{MyH2$(Uy9IlNDFt-k1dM&9R{)21ZPUzDTf}i`Y3Um6GT8O41GhOaSo*6A z_-@X1TnzrH5O&x&*h+zw2J8_41G^e zQaQcVp-fQ#=%upB_Frqz)YLp!JyZX*3GTEjnaO+#6PHY(lYEzPjvcn=6)L(IQ2vyu zh`)F=9#&dWA?j^nN-yoy@efAIs8#FggA4 z`*M&x?mU22fr$8HFKF=6Umk^-oB&^sSMfEMxood*l_!D+=hrzjhlgKXb56c{QXZ32XG zNS*G-%&U|BhzNi{lbA*OiG!*^n8bUe^pa2O79LFc(Cwv5EX3yADgd!Y%tetac&ZAQ z(j&^axU6_Wfw%w)J!rx@dO!q5{&`EVJ6Gr9_*$LQkX?HZ+_A0l@XW8c5+JVLu-q&N zSq}pG>%3@mNi{@u3@EvGMa#RTxlDdvuQo~3y=AubY{H&f>~KU+{32*!(RrjT`r~uJ z?3a5)B7>;c zi2;B6tApi-K`xbD^O*Sf9Rbn~S&~+v;1lmfV*(bm(&RT2+ zm&1!ExR|2;K8Qll4{gMj*G||%`6r&7)2m^S;~1is*3Du7YaTBhZIJeCCL;tvtxv}BfSkj;)j8$xyy+|U zL8QtfwxI`X50-)|c!G4Qx+v}6BmRf|M%iMBY}8rrhQGMiRacjowa9pdI<-D6L8dY- zlOZskR+9^AZ7x>+f3H%}(10#VzOv{9zRxVg^O%-|{OZFRZna9NvZDCP;h55Gv$pcI zC-^e>{J4s_%PIa&4}lx2n#|NMUDe5}=NAjsM7bmjhPX_^NNVfLSJdX7iQF5Inqau` zP46ElD*{+q;Kb|6QVpyH32)kj8MwYj1EpMZFo{XnRtJQ5W))ua3Wl@XZnfmmSkaWw z%=(L(;KYNl`lFsY@&ega-@xVoAj9+iR(h%vE3+|DMm6hlw1k|`j2B!# z4?B@Qb)%#|;p8h0Fq~MY#y-eM9m#sJ)+%!u1Mm$ZIE)7;H7G`f=o`vEL1}^qknNQm z@JAoLcC&-ZuLF|u+-LS^Sex_Xgd6z)5LhW#?);b<|5&FPl6!91mizTDV?FoqKS&9$ zNr?rf)x_vkN%*^sK*&>mw(aYxa*10|`%L&b+!%Oo`3xRlw`=XTNo;VA#L0{`rpTS7 zHpx{+Dvz1g7Yzn8N-TlX%{RTG7Q3tjAZR-DKJxDSrt*SD2IvzT=;X{c*f5_wh?mRt z=LsZFE3N}KJRQaFC~A zII-Moo6WZBCibK*i1^7w7vlQ*)DhB}i*=NILs;7Kwldx9PJjQ^F$zV%ay&sn)QzUk zav(zn2=-0DzFhmWUR+yA)D9dd8st4&cV-CZq<-#sci{flxXS+fq8I#@bY}AIfxO-x zdEdJXZN17vLQ+<8pS<3BdQ$(_j<}gA<$;&Z3Qf=^$)+z&{dZ|5?iUs|2=cFP>4$}? zhJV_P0VP&n-QJUIZ?t?DTg8b*>&qb=k%4!P06=$~)c_Q(P$jZT)Tn#;)6dbld+qN5 z`E`4T?O@LeB$DsQo}9FVC<3(I#kaZU6?(zGvb!$$qG|ZBk zf$aO_h1;H%G(4MxtN7lXWD0?f0mOX4cDYXy)*NSbQg}5aBnQDrk}oj|R$qBq@IgXM z>?DWyD)`sc3%^0+F1K2m*KOyJMt-{ea!OF8a{B@?H!=U2 zw%4OO$6P8%OjdyU`nZ4lM-yz5t}BB6xP4a{74AUzwCNKBLUIyZ*V1%om7xzRa}1}! zi$nOn8lfnm0Xv={pi*)vZOTgtjR)CVqjSof?u3%QSlWD1E!$TQQ5u22a0AcojVr%8 zcTOw!*bhc$KRN?9A-D}IHv7SD!>}kDUS?s$ zTM)Ku_2>J78!$-uUswO55Clgfk~uy@-7Bd+e>}=Wljo1WS5FYHF|#@ z03|LK<|JE=^zVO1)a_{T!QGgwPsDtLaxV+R?5Bd2F%VcV_ua2o*orMcs^N89(*wd< ztnQjHykI8|uiZVK4DQWRGYj=klbn}u_|u;T&&bt3Vv@UmW0je=tIE<`to;F+pu z%h2bU^CVlfcj||dDEOFsZ|>t)g!>w=GABwykGTrl~CtsbA36gGq=pxAdBk3fQsW}J%y;D&*LiX1LsLw zmhC)KU=VHs)m9PFCuoAmS#;=`f*6F8>V5n4_Qttfw}OtZF&9`7>Z6}C$XLbP2NSRb zz(0ev$3B#1kED&l^vOb3c-fU54{)Q1K2{&0xwA5`D$lU7>a~bXcq#I10^W@o(9L z^uz_okWgu1^Xk&1IYj}Ee0RODXHm>;xvV7Z43d1a^On7CyODd;2oF9(MsM;_DouCW z&LX6d#Qjel62CH4iJ}gYM(*>~^j{}3FAt@HgrAg~I(+sPr)G|i6Xn7hx;XW7iPlK0 z!*W6yhq$4-e9zgnjf30~szkV);iAk%efHldg}!DCwIZB|%WAxePe;srVvtAOXWwX( z{H1-+S++;*@|ri$GEvgL=dG$W6b#w(O*4ITIuRl$9lf!M*DOk++lgPcYdO$%5Csv*cuUk zX}VRRSbGwaLN#eAmVTU4cV*DcGI{W-Q1@)yJE5X&?Q8+5o6PPyknA;^+~6!dRb&7+ z(&&EQ=e@lLX=y~|!PlS~y|kB@Nx>4dDyC2?B-I-0*eW0Lr&r#TFK3L)td~~bj+NVR zEn^~fm1BZpB`I5)xyxTRPujn0XH=0VL7@J1ulI=jOyDA5k5|p_uAr!H_y3Xo;NmRQ zO~%3|P2m#`{Eap385C{xOdoV+aNKXh`QZMyDq$O$wPHj3`F;!}0qGy{U?W>el3Art zTTM4O)ilbG$Lr_`nOGqU^SeAo{xtuG;@|_F*T^{^m;z4Yh%72a-UbFYzkE~ax5W_C zOU92Gs{Y=E(1k734cq>by*`B8 z{j4Gl7o}yiXNj^HA!BWWiJx>8M<>genM_wq* z@thtcu7&H73R|i7#!>w!c~4HhxQ20tzJVpDhpO#~d+vezjGP8G8Bkj%5ig|?Vyk+ySsKgUbm zHFBEXf^*#4w1~vzkrW6^65vrPDCKYkFV=cj=%iuiI`vA2JtX8Jl5GF(XBq786LXd^ zPlwjv?#-H1zBSI_THNn7%lS%@9{A0{B6Z!HSlrtohO&L5JgFu2V8`sV{F1!DF8c0E zS7*u79xU)%rd|{eCd~-XsFE9Ac@3v}WZXR;3)1bX?FyIj4lZ|?&%CDjf?JF`92?^d z#LRCtQZX5fkC@3aLq7UNtL(1fe3%0=ovC|y1awLB?C$t2;ORxac%hdDx`-2cI)7G@ zi{K3^R48GwE+z~D*Q0yzcZ^pGLsxS1Wt(yS8tBW=+u<1ol8$fic5|9;=UpSsx=bvH zZF-r+jR*8BJ!8X$?K1W@og>243(UXn;N8F`&dwDZvX@x<}~yVXkzn%T3M zuMeKiHs{Xj0Gsx>cqgX4MiavF2J9EHaO8A6NhwLeZrP;8yNg=)nZ#4j%V%EC8!P6@ zjQy(8YE9$N8MM@8Y;iP6r@NLMX7p>U*s8Y#w>p0$M!~QBMEzEWO|fV=UgVt+C_*O3|Yvg$y&hxQ5=oE5hiTkUR>Bz_M>tr}VYK)9oG|-EMPj{Kq}{z<2+9M;#p* zanf$%uxF$7Pug=-1IspOg6&`0#|}>)_uQjTACP)6uWXP1D;Lre4-$Cz_(YFFUPzA4 zj<^l-?ChEaXo+g9(CV6M|0RHG?f0=+js8^*j7}a}!9Ux2Fm|&_irOn|oPwuu6R8HCw${cpl}9SkgN&B1o;^m}tjuua?In)o=^i^1>jD(5Q6 zY_%Cl+)0gw7kh@5{)@gKbGYX|s9hE7QHkUKyZ@Z)IT0wg26Br3weD;}6o~%{24K^) o=?4CTKL_h?C;#c4+XPM%39I2B70CjiCYt<8 literal 0 HcmV?d00001 diff --git a/docs/images/design3.png b/docs/images/design3.png new file mode 100644 index 0000000000000000000000000000000000000000..d544ed5a711e70352d367a14ef8633be47321156 GIT binary patch literal 24493 zcmcee$9Ks*6Y1sp>_xU)h)oEk$w@Ml0k;5mJ5R}lbyf#4*g z;|c+ROa1o`q^v5{B?N>Bgsg<9x)ZF)1Ol#d2mr=Nd!%`e}t3mdw+CMWNI zcD$M5+!fuP2kB-TL3_f<#KCL?T$#di*|h{`YR^1-{V#y-Ow`<--Hlv5^O)JzLI zjG~{EH$lk`Jm<>h{b66J{ zEfx!nb~{-e_=-MWBDsUH!%_N0rNrir`+|RXXebLDGo8dmW<2pL+wYIPc(`KDM0zcD z-ZuN_tIl)Jn}epz=ur1e4?1?r)Z$Lu4s7a|23I^+mWV_1s|id(LU!9jEv}#K9Tt_` zhsFpXkama9P)*K6nJF(IB=1Nb8ttYo-D|u{|)CB88+qux|AN!!` z&E|<@okq*WL`-y;c|=HZvDSq6xVXrO7<%1rtWn{~LCVl3amnwM4kuF?c=eQhJ2SNc z5>{5S-z$CvjpN$sf})JYd!R|KXY}xS})lPpXQcT2J)2BUE=IFt6W`pbB zzx~haM;Lbv+EI{@kPs1VR-4M@+ZZty#tugl_X@&E8X`RJaJdim{0=j%^rx&jUXJIA z5?dTL__LLAB8w$|G+LFpPgAuh?j_=dtU$w~&L#+bdD6z)@tu@s=?Y0O=TXFx$mlOqg)*KZcQCIOn~mp4-Y2L=XGluVrB@@7Vu$L2dNGjomq+bgO6vvWL& z;H*8r+c7#8R`S*A2Vg|MH$Ts~b)4ITD9j08_ChD*;chhBE_+<87R;ow7#T{J;)^g1 zoVCj)(-aeNS|=!?%~i;l4#yE!?ror*uhc^$=5c<)+#hGm`U52y^l!{3NP!dSEWgi| ze;uS7k?^~LE4(jOo1N);FS(s}2YI%9Ilgd>7}}HHZTY`_nMh;G`*N>yb2xQDEzh=z zp)@%1HX=@V!Rq;Q-HTHsBtxOzwZ@fLO^P*rg>6hyXmOu9wpPz0| z{9hm8?b&&G@kv7SfY*lI^MAfScRiZ^ax=waZKot~H8>pm4hfH0zs<#>0?C%yu+wXy z^y9Y+wy!(oHqf7_A4bOl}?_PDdd=OaY}e#gEC&D)hG%7}`fqr6vqJzCtIF1k$sv-wL+NpOgQfP!t#pv#BL zakEpcOtH)F<>}@3R{U#G3Zn1C(M+C_qk;!=^>I2_fi~wj3r+ahDObQ--P!HW3jI4y zYZ0xO-|l@uuw@^!;V=ZAj*Ea*vDIjWmLpuFir)BhXY8Bz&7s9aT4ELOzA50e%%2+_ z=_!njk56Z{pHP#nD?oQh7(X|=vJuI~gX&_&RFOYWh-siPK@HH@11lPzZ6F*CJ{mY^ z&lLo7x%8FK72;|}_^r4v!L-fgzzanHR%kbwX!eV*2+?aWC{miYrxSm7&Sp-CTuc}0 zOTUMb&?8kXVzt9S7>e0&4CGmx>rv3?hUeBsoryT%Oaz?o=eyIA$};HhQLMFTgjxES z`4+N&rw}I4Edrv8baQ)Yg0p~vc`;u}_{D83iCTeN)Yxs$n(Pv*i|6Grg--2f+Yw1d zOwHeud(28H9Y?hsPU;6Txcj-0-R*O|KXyAdnJ%hMYo1E4F)tz06Dua>L6qvP-sR)& zne8b;Lc>2axAkCI@%oh(H61(>2i5iI4qAXuc)31ni#)S*gR z)#A~2j-8@ZI{75gWz$=$)M7RP*YH{Dx4^nlXcN@}=EiriLPMo}$6lF1r4WYN{q%bz z9O(y-7-=VxS!H+H9O)AI^cs~SxXI;uGiiq%jzRX6fFrQm(R9w)Vx_b|q5y?RFo9^5 zPNNjY`y{qJu{>ITWx~^qhpWBCj1sS_-AVRUDzSYhwuFL1dd42$#=LD0y8KoCn%-4y zN>C(JP*6~T)v`>4O&0-QX$Z`iEHrdMrODW5ps<`2vYm_S;_rCDg!k#ycHI-=_T~9K zA#6!i1rG{8YzJo;UinL`1v;ljl}8oHYY|dGo(It;+K9#a^ zGV4dcmC!F#XjB5DVJ8UR?|FSf!a<@}>9>Cs8|L22iD{A#;S3&Y0p^3~DKHHYhdu;% z6dk&)H}FlqM!#*=gp!hSu1LocN#{CZH6ZH>R>b(YdZ}5o%6$0(b(ZSFovCxYil6eVM zWDeZQVL90w7XzDgA}B~?oKj|2w$%1D(qz0&A(MSLW!+(|B^EzF(C3aa3{$`G=935& z8K{^AhO8eJ2^T0H{kIzSwbe}b9F!RIOnH2+!|g4sSmP8_BaEr4J#K^zBA>vtxH(K4 zjBL;I2c}~Kd|2@6V2rSVw>fPvuB04}6ybi^UYo`y)i8d)IvgeaIK|>#+BrHE60SPC zC$A*lKCf{)NWf$s1O>L1BA>i`xg0cT_Mv0~!$Mwnm`vn<&!Jbh! zZpCaO?~#N1Lo>wDNCYCDTkO|%Q;XWIKl+)ArdYxh!F)Cux5(jlP2Xt;wx7GjdQD$Vl zop3+V+&Tz~_%@diu_eyE>##Ze+Z|ReoecUugyAcsJXGLYrEkl4knLdPG)@;nchWt? z!$L8ljD$s|)8VzF58HP-D&NISU^%<@IaS>7gTM{wp)>Gq5o$4VF^;H)91n{8qi226 z9z9|-_jc14D%@^Ee!-Ju4a&b-xBNaI<#=xUTYeySHEv#+fDck9X)>n@Ohc%Op}XuL z&z0dqH{L<2JeQ8a7fllnM@1oEhw;rE;wmMdg!Nrgi#bhv(m=55Rxx7-W;sYimpuRV zY@y;d#(=qnpI?;*_3CXf99qj)fOO7KE|oqMHmRi%DmL5aUD~yY6tm8Pg3bOn%$$_|>KC{{lVi5m8M}x98x&5Ca-r8Y z3DnPJ-of74n4>rqg5{5@kf4maRIg**!HMdBxBE;;$h2Ly!0;}N;eKe0jo={H_pw-otr`hwvpVKQ0MTv`KB zH-Ua&S2ozEY_e=W)YAkW>a>52lqF-$QwU#oktsD$bXNK5qa;*zuXA)?K#JEA?Ly_W z1&4?zqj}TgXt~V8@H*^H0n%9~dIPE+3_cetsWHYF(+><&VkKmqbHRp8p=yf76RcAa zf}^uub1s&5MpQ?Q+GT@1nP}LpmS@t_BipwVcw3#`9DkqQ7ra*L*Rp;YEy3q%jbE#i zZ#$>fT*Il)9WxBV-ur~{mM|PIFvK_vL*gw8GQXYjXXmCUa?%`8Gi{Lt&IXWattAEdKn$Y+|cuihO;Djw%_dOZOS*3gD4coE* z3SN%VuksY28^qR0!GKIxJnUU^B+@^=g&{hBv0;g3>qrlIj9IGEb5#1=QbUD}kDF$W z-tEPF(_KTvDTgW85|V;B*yWUh{l142R3Ykk!!F43-Whx$5@k0`WN z$DpNvB+AIt?zAHi2sBHzTbJ*oJkA=v#_zbF3p+*J>b3lgP`c%C_9EHhP=MpLU8)H` zw4co6t%{X4M5lI|J}6IqkMLE<)#D{B_HbGdE_Uy|BDwiq=@cGI{YFw{HMwn)uG8%b z0#r7)BSLfp_wQyZ1i9}10ZO7<{8qIwqZzXN3n%WOE6X-Jp$5Eit#3M1b$E`6lYLBZ zi=1qGZA*UeD5GEFkAH0kTzv50usxxIdVVVkXG;j=YjIdBc>ghHvL=a~il6?|IyMQ0 zi7qJB>IF$qSHhf1%sN8eT{#jXs2|NlLQ*mtH;7gj+k^TOg39qhUGQBuzm0HHev!O_MV|UBl{xuR*T;m!D#nPjXNU~f%e*b<2ADbMpXq`H$@~Q_NP`>)@&`mVjpB@BsiU@$`~svcXvKI@J;2vrTXc~7#nx{K29OY zDj;`Z_wVdIZlL!g+;PG)0QW7}$)%9t$NAPU!}h^^cC@;6_M%*y2;VO+rge1^(vYZ) zLo_xAXUWCi#EnA{aSp$EAHnmto^2=M4JKZ9J6}1ci;FkH!XJ$#FB385W8e@3PSN`Z zLP6?4E958M`Xq%&7OX$q zNoMT%7~-j}WJCTRZ2%QBLy3iyN7u^W{VnYx=$9@qA)u~jSlPRO57A*%@gT|jzYl2@ z{|7L<_<#cpUxSK${v#3P(E;ctr@IXEuz!z7hv_5&G&GC9sH)({}#euZ=L z-vNek0q6zkJ!Iv%|NB}Zrxauo8SM{IZvQ&~l0_N-5EDiT>PmmxPe2s81CT_)z*n;U zzj35fl1UsNu*g)r{|~aX{~aiEH-*9f4geYxbc)++JgP;k|BXt<0A@eLr}gpx;+8>+ zg4guBH{DkWpPR#2VV<{Xx^mCOikuANLm1r@@Fz44y>G6ydHm&w?C|-CL zmY8$Xv5WRG9@dB^Si9}>j8Hv~Sx#-R`*=WpZ=P1|>ciSoS?*8GX5Fa=uXNG--@o$& zoE5B(OSB9>Gw8LPw>q>*k%Zx?S8N5~O?Hv3!Yb*1b8=WsnW@yQeRNB=GIQK)aFS_6lF_HkU)NyA}$$ zSZpNBJo;aD(|feVD;-BfyxNS4jO?VEP#do_S|X1%5?i6$raJR;=G*E(g(|uW00w5> z$JD5o2Yn)sKtw|;>!{T(@U0~=m|TP*S5Z+>qlslPs;T-QW<7VpVMW$|x>$LCeBX33 z{u~`&yET;oinxr_j2FC}6{jhlQ>)eOjtJ~;+{hhEWk|<7d#(i-mFE%ce#R9$Q+Bz=GGFs>qUFp-`vGDi`rAYUZe6rG;UVWmjDP+}C{Nha6n zCm6};e3gQqH16dsFaL-C^$v(nFXS+OV_cj<@<4P|9xH@BGjFihOP zbKN0ugxxaG?JK@7)GORUqI2G%-EMI{7PJ|HL}un zEX47MbOJEPUsM5z&4(v)9H|#T?vilXsORvz>(d|37KkH*qjBU3ClzxA2wZ?C=}im) zpvcR^uu=<$jK7^^{yB0cPxx)@eco4I=b)@~dX3YI)-7h0!jP^(%zwOA1Wsxj zpgEdb$|k^nj| zAp!kq0DvShB!aq`BgZ%=?*Mr7!^r=o*~|IH!=<(#o?Zul9$kWjc*u(#p1(3Vtb_o3 zF|9RU(?R)tyadJsgGsxNtYUF0o9Fk%S{a3k*c%|o0lZ<<>7|k`y}wu~t40WqN)(Ws zucSU!SI4>upGYmAR;6431?gZS!G0aF2K3Q5bgCf%?c2t3k4Jg_2_7G&;rV9$?Cj0~ zqTyYEpz&TAvPruFg#w^=5{&z?3?nQ(&!&kA4*%{dqnDt+V?V|}_V-DO_~|^FK(4(v zo9A|OLUl5E(sZ&Dn~1-79O8MgD(R@ORY{7`dnt9)D=Oa)!`L5%L#`?8zke}45Cb$DEq<@!X$0C;Dct|p_C zbrbS@4Xi@BEBxo2WAgyOs&uFUWc@7N4}}0)6ks=W;tnPz7RT*|-d>H`cUizoy` zTKab?am`?o&%h!YC$yUD1!#7k9=!{|qTGg79%ax*@(mWN_0pvQEolNYD#u_It!)l| zb8DQ^9SlwmtLd)*5MHQX&~lAFg8x4B6?>!GgSw#7akc3auo&wYF}=@M*dz*w3kq*9 z*3yF#5I(Tm^L?5x?(G$!5+}mP$m|cX9Sx$GxFx#MMo}rmWieFP2XKTNhCq-*p4Tn{ zC0_msu!2*ir;Xww(5ImZ%>z+W6mv4{g6qNjQY*`bpR0jZR=}dA7Qn4cdMy%6p@rSA zPiI<&-Uoo#gD&;CJG}s)r8b-Dg5}pc=co4&m29|Pzq)>B5K2HPaz<4ee?CNlgnftr zYQbZ^XfT=9v)kyyf|4+kPQ!lppPzekz!^dMGM&v6Uv~v%w8vydAm9uRV^i%1I+BDI z(8Hren+tV}EY?#;Q?=|cEwP2nqal8_cp2iAdZp$RetG{YZG}1^a-_^QhM-NJ(3hne z13j=ZIcK$GG;W_Dz&L|IhQ3!r7?e`J0NeRylpi9Jr;Ppma$cV0%k$NUGM#xOQbau? zzlQ}Gm57y)|EsT$W0D|%9X2{V^VrPbUW8NnE`aMFIe7LB?HzZ_E2$^RO1LihR|nEq zBostdQei(|06@2Kr9o8_-U)Zpo-S4CaF}Ib4|qf-^n-cEzY@~)wxeGIC3naF*2wY! z8%Y(ysdpwH4kHNZl@1xEiDo(5rS;h*n)dtJ{lyvs?tTk^u#bNs z8wx#aLsJRA+%81Hdf)CREA@8=l(1u(eJRjaRnpdprK`e{`3uO45ul#e=Iir`X3fUomXBj zcLorl2=$AkP^N68Z%xOR8Fz|S_=vxr--m^lJAN6Z~%q8>2X)R|Ecs~(eR00 zZbiAoA(DzorkmP#k)mxpY-QCXNn26=A~pymleSQ~Y#ztB-H?7u-z?J=xM;AzD3d|^ zEx@B4nV=Y`lY>$Q&jDNIqK@$QuPvi&-gW%~(<`YynVXWZRQeN{vr`jdOWs~Gr~$Xv z=f~>*P8=W9^9PQv&G?*U-qq}?uyv*>0>23GR%U=t{BKS4bvx(cIOE+(kEEl zB4Tedt#bPiFrtZ>iEXkb+m)71ggde6(9Er?TqeQyW(e_ewMQ)kBs7CzVEP3&aELgOCpTu$=3pQZ$X%5c2u0drwp*RyVL_NM55n=x`u)uJyb z$`TE_o*&gyMjZnuzO>0|hNR?5FIx2zqhI;OU?i4Oo>28}CUG&DhATYgC2T<~?|Fu$ zXjEe*#=J3ebX0}jx;brj=xG8gi2WvV4NV-GC<s8j zgj=yK3~Q_@tR#;yA(B55zqp;ulk3d*7G+*;$@Qw_w{B|Zn$(&8L0W;?D z<<9%1lLdpKnS9ZPQIt>mfH7#k6w@Zo@7w+aSOeObII)7lNy*?L5#}dc645d-N^>kC z_6CwbNlzP>xkC2gbg`VdoXe{3=CahEegUxGMB$LvE**6B%!wJVg1E{#E%1)V!90hQ zb9mkzlnKZBLFOIfeG4S}SA8-W-RRGl8tR4CPBv zln6tPZCKeN)fkSC@uAH74iL&rN~|lTo1mmM&Di6^V_6mncIdkVy>b#ovXE&8)tKgw z>AVcSaPfbDV4PND zaWQcgc(|9m;_8KGo?)KbYYS6{^-k|cmy92KcD^j(Gzfh{KjRWE8MMG}Q!Cn@A$4F} za`2u5sDQ59Tq)>aICjG1`{~vX7gGiOb6&7!X;`>)MkUS>=^VwXJKgQ|R51sofnd8# z4_T{m0U@I(93)&$sR<;?(kn&C(B$DP8H-pA4N{v+=wfea&zirHrJdc^(?73W>B8la z5H@h#Df;(_1#3+|`R>>uQ<2Dr9SFp88Wwd%tG@b{%S@Hyc6kOu!N#jwTczf%F}UCl!YDSIB$(*s^_IlY9Cz5l;ez z5AU_V%@xMu*T(O~7)NiNxJvsQIx7zb;mJSFH%kTxS1mc)c880Jpdan_@U1tv z%$qH?M*a2+af;ij0l$Q;;$plZg5-H+Oj$$$K2nXRQX*IK{CNHJvMhF{O#!`gwp?ov z@7*nvSa^53#OzhKbJXL$svz-wJv9z4#fdvghl4 zL%r#u(ecp+o4vuZ&E0W>ABEK6tFTd^#BX(v2v%Bex;j6LzoHi%(x`islhH5=_kZx_ zpAKI0EBK`Mp6)-uBZB_T3_w1cY~T{I|Dchh1TZP@wxYhv{*M7|$B~msc#Y(z1^oxI zIxzq-?dZ#(`u;yI^&u<(eTO($7ZLM+pK~eo7kyn-%jEob#7}$xb$KG&(8>IVmEQfu zO2e%b?*AR}|LIG(u-57+17ab^Y}Wv5=ggIXjg6k39*6<_Xgp%|CNWP+&{I)K{F;_U z8U9D?*>!*PQ#VbkgQT>i#aPl%Cp!A=-Ohvlz7wLu{rQiNnuqtn3=DjBNa&L*E@#==O%U((=0OWp(UTGR ztB^yU+3DF)Hs$AgD!T(rn0dm#>}9j}I@LCgNboQT6ekqkOA|w^Q?Gi!yx{|6eA-eB zsHJ2S+-|?I<)twg6?~2tIt-bG2i$i1{uL$pH2RGqEu(Y-!8ZB||6)D>hMm2v-yDXJ zMdBK?VRC=!&miQo75vdAZ)aD9pzW3x(Ayi^Q}yP(acb3m4_0rp@$H--b0zyTSz8J~ zrPmVjfd&gCk5oAn7O?4WlCG&PvaEqiiv8VCmYxhR&z$y)4+1#vKC89PJ&G6`Z#U%O z+OIuX<#j_xrPCsGeE3@zMxzdd?SDK2lc$LWxpk%y{n0q!4p@inu>3DV2Ii^bI45;qv0+?i_l$b{1X;-fVh(+%LnK0^^RHpdr%vP0vDL6c zt8`u}HEU85tV{iiY1r7^-vCov62NXf^%u8suFLW%l{ zfODUPh6jrR$pE!BRY@%VR)FT)2P!vOAr+JVXoP=1unW|E7*4;bOlWcox%3L?FOe6TyH<%F&iW?k<0afiD0mBGX5uqO71Qd?gg&^Rs&{ZR)z@YvIQ@ae@i>B2 z`I$&5{lH|@GKs~cmjMnNzn*yFV(|6N;@qu-0iXX6$O5|kI>?9)W%+p*PD*D$=rly6JH;QlN6_~pfV?Tu{e0|lYxRTk4}>wjr$rvV%wnHvxzq6W{MT|9`*w6Z?wpBP+OH0ypm z%m#4Lu66!tIv|frKW2NJEqxrx0Q0a9^!JA$6PPA?K3wh~s{`x80hv~*QE90dxBp*_7)1uHEP`4cw@8SH1ngGo3IIcrkP|E#43|vv(Pt+VPoKCh z9-qy0u22$D?mRq88l&0smmyBP`yBN`VK5#R3CSj4;{qsvJ-o#zAD&QE4u)kd#m!Tw<0d07+N@@*Jt{K-L71();(lfzXSUT0BusL_$8d*1-Kz zJqyTG91r@xz3^cDI=ydRHe9%JfRlR4`REzNx zZ-Co8QcnPe&x%M@zf4Ly8PboH0tuX zt57c&)9R%axHx;yHL3vOr4U1MrKWWs_3z&Kzap8h@1gYG^jBHfLWdYXk^v-x!xuk{tG9W4gr5D zF?!M1cfof`fF6K;5)qWhM%=B|E>Zd_cXjDve&dVtY+h?^sVK)HURLnps9TbPh zsROv0dXZQtqIKpehFbijkuFw&kDP_^ARtl`ZGk8mJ5GUdqLFCtXY>dq8CsRX!59LL zObP@9gg!IlnOs4CfRg~?dkB;fT>)fTG6EU30I+HM9jwx1t#V<$Er^nsWex#zj!C|R z0CN2Oa|3yO7EIRi`tnDsMn98#z(kptD3}K^P1W)bDv1F29L!i1wvUZO?-tQ|()7{e z45#@Wx83ndeNk8{|KGG2&@6?6$Qx-sW*tjEhm&Q>vw`EZ0)aePi|ulpe=b1sUI);G z0q$G){H85J)fyy{MJe)Lgc$#iDe8eKbKY7+1n3FN2S+CHHC#xPE_yPU)#1?8SHDt| zNkQ#7yUZJ_7>WQIpyZ$uK>+!#@KvuN2p6;i2o)_B9R+O_2p}drm708P5;EXsqjZMl{a;-gu<9>(xI<~e)Rt=I$an5@eZJ;RRW?g zrr^iIHh{AMS}@!T%|g$ClWGN30? zVTZe6S?dqVSCu3sAaL%5dER+%Otxix}{pEIs=)Lm;8XX??^f3()N5na3 z8yHW!!f(&PnNiWw8HrB%0oa+PJN~3RuxB+iU7SDk1?NndLk$lD2vOa~ChYTjMB;bh31alg{~$9&1M*2NDMZFa4_~Qbw67=SO6}ddU}#@ zHX>2)@o5%`ipPEuL7yt!<_gbWn?Nd#lkIN?$sYNJM#qgXl^8aKnuxy2KXKS4`Y87^ zsxaXzBx^+s@rN`VLhR}VnSb#_2q@&*7v0@9LS8V3@()t7ahdH$F^jUxZg40r|m<0~*T8C-t1!8*LU4 zu~^9D5qY?m0uexIps(zPr8ri|V$a8y#tWlOhoVmI_h<4*`x^}%IgNWElIjbC{h^EN z{ZT3(FiJS&{*8qM%I!lPF`j%IzW828x8qhffWO$vBF1I>`z0%{ERD5eN@e7HsNAG8 z*!KzwbO(hV0CQ#(TrFR3pjB(I*5(?YoYP}Kq3)B>)t@iR{F1_Tt-o%XHL|QTf@ORs zxYhy2PeVJEo=ReIx4p$48nju!9rvNkNrzzO`%B1ytEIecZ~QDJ7x5cEQ^ov;$FOq^ z_F#m9Op;HDVe%+R`v0{1U66wOKYwZD3n#=Nr5JXN+kW605 z`02BDZh&Uvh)LUShKD{>>-=os6)wPOXIJt@EJ^1*)jov+HNhXs`~JaUcy61|NOFg{fYV3$=Q7dk z2Jh8;H^k&x4EgO`SjPA>!JC+GKATSoC!ScWdi3!VG}PpPONT{w0x4c$QmQ-F z!^38$cQ6|l!^S|5WtEH|qE?=uB`&GEnWTJ^A;H{*vW=uFZ9k+OFvUCVR6yZ#Yby?m(ix_aPUc)O@BY6QcfNS|5>s z*bjAaE_4=&-I<%x6p6nS)Y%q=m9F`APyC7)GxMv*%$*#tXzq}{P8+(P06{$)ve zA)#oaAEuq^nX_(mRB&~214mFeqLnG9!ROB69=7t0ifmR_!JA1AMzuZ^9O6XS=t;(g5tMgyjU3b$R%~$Ng{#|e9)sf2 zsAx)SGhz{}_E+hDqcvn^6al!SxDNVXnH79ufPgHW{cXO}@kxr#h>H|HNxw|feX!*2k4=!4h79ng$^<~B1@U8hJ2lj8?<0(| zF<>YfYc2&knVtc$>h7o0N@TNmi|7%c;5IU#>xuS-vc5=@e%Z%VZfI2Cf5=OuM+Ck- zjcCAB%<;I+XgpNe()ECejg|lnnNB|%CB31!mUD|rGeY#LPN_>t6!EIyT%(nerUs1X-R8(nWfNf`sFpIflaL;Lp^jSg$%XAc~C^_9m8Rkc%ip~QyTPn~dm#dd2 zF$3J^6gV}Lxh^gsl(JlY!BeuRJFrR-A-5fv0*_HUp!?6p*IzvkE>FQ|APgHY5OV&z z*q@}g0gLyN;!jt78Yw0HU{PF#$VE1vYpFvXMgOYjmizrJ0&3i%As;{F=Yw7bqTL8- zw2JrAt2^I=H*f;+&45@!?7b{AClh{3$|NJON8wMRG)W&+BzlC+<#wMykaRgh_|RX= zc2M|nKbh#NK(S{R7w%;@ngxg*NNfTQFUkTlW5ivO0f-iZ{J8X-FF^ERif2pR>PRIJ zNCV``3V~9^#kMB^2enQO@|nkZS0Y6hx1`s5{7r@n-aYdLg_Y|Jg{6ESNS+GkaeD6- zv*b|q7w5zQ?CBq@lQ9B`Kvd~SRDM24)#9C5*Y-|n!^n%%skgZRe1o|TPscsVXx5am z`C%YdpgL3(D=GuD_(HAd;U+ z@evcNQbBxXBjvMS<-D^7Y36S39Bv?%NV#*S3;Qc!|e)+YtkH5q{hIaz268a3RZ$QyqfO?&*RrC`R z6Y+6Qp5{Pv3FnL1lw8V(=YrHE&BzE|3t&!bFChQxvDN*>8B9)cM0l__zJN(Bm%?m7 zxPjD2#Th1z=H81K1&5$h*qa!}1QpW(bF3#57wg_cOI0TqQj%7o+(Kw|fOgO$o;8X7a^kFt> zt}iK5AXk;S=OGtA5(P7;CSt_=sUL}jl>=yT1@DdQx4OHy<*(I^_*vS}LqpXBcX6TV zmb7w`IQqe89x4&Inql`|5*{J}e-M&kDcsHvqyC`bsfB5)b$I%Fyat61dz>`b!G(bA z`zikCRC$p=wtt$8*H${Y#V5 z4UscD;^DLUv2?^Cp&;UbIPq1$7OeP#F#eNw>{GgUi8R`akgS!26!TSOJc3t@YLW;j-_LiDbqo zA3`wasyx{w1jw}A&Gx-~%of(IGv1LYGRW=h)Eq;x zKWiF`5*cPBQ^p#$#KXyEh}ru!`L2vUKJ3Li&-5)O|CPeL6VJdy;QD4C45l?ej~1!Q zF3;12p+#Ml`Y}`{g@vCS{cTJ+E!e*o>%Af}F@H)4jV@W3W(7hns>{hBeZpL~SoC4Z z{lyhl87isJ>3m>`tbv{cZ->lmGY+#RqI^W890jn~GKs*&y2v_`{V;^eRcR8a-O=f^ zI#uR6uIs@G00d!|RW7!!$1Iu1#)5)s0;!?bsh8_^8Sm`k7O88IBZ83idv?H(fEyh>Rw^gyD6Bcck?A99iv^Ikd#xW0FT#emN+(`8MGrDYB8%38E%r+78$LEf>CbyTBp&;D=~?+9Y%C9b@TB=JCSQF4l6D0; zz}V^>D(kN%lq^6>LWZM83X!{+NTCxKOymX%3Hw`FjI5hko^_ATi|J@x;+(fmoPe-(v4a}UT2oB67GWl@ir zCw3h*8sKl1Ca4SF-yyVlIe#xB)u>i|eC+Phzc6{j*~>N`Bo%alv9;X52r|3YNJ&1>kJ3YphwV zq*w?LBON|klUPWovm1l5G=I7~$^4+?K&n;$_=Z$r{0d7w(7*q7gf;xV?N80?%pL1o z0u@1b_elJCI4JLAy?kk-S>HjPeHX;%dJU3TX;b^tes?<8%6CI?UZV5uq3rTf=CjfQ zH%WquZ9YG5uwiPPp@G})sM81P+%o;)3e%Bb2xWF_R z-azw$65s}9KUWBAqKf$gAg0fXkdVmE+7>(zyKGHj&%R=EgrPw(h&Ppv<@igwMCJ)_ z*84*!IDy6WH8Inu7KPjMFmBRWBFrmzWty10Z*LS5;44mo?F;raBp_`oB>ZMVb$E9) z`RTJ6$Vy^}CR=kII|)5+UoUlWYbz(~vJDa{_+x9x!^8e~DyR4KTTHeKw$G_B_vd@# zg?Ve0`H6!ILd&}xZ!Y?&3%q@jmfoc2#BT^llomFr9MT@H_9{%OV)&2c^n7Tc?s9d_ zah=I*ZVTEw-!6xY70MOTpZh)5T}1tkJ#a0}UpJIDjt>A*8%+a`lD~O{)aPWi-2C*QjC@O_Fkn2YD^+!vPdRZn#iJpVyh zx1Me`Sl1%rxQc>(lpJyCrrAtL z;{QN~X#+ZQBB+!*nz4vc(JwJe`JuLT3Q@z!U)8|M$a-kuTRvP)PoNarD$$@GERRHP zY(TK5mKtDzkDCbW&mlSf7 zJTsr7eAm~f8uCLqU?`h+yPb+}XpphWx9L~^CC|2ZQBe^sCPXMQ0e?x2B1jh!Ldm|yWR z0M7;7d$wT>A?20~vyjlM#-S|Q282cnnrekUswih=n3RAN*bIiC$NK`cB(I>Xk5At$Bn=Uc zZLzYyTsZ9}Hy$917lId7$nR(Jfge7Nz-dnFScf+H+=T(8O=g@8VzTskL^t6rK{ees z&%%z^_!i6hu6Q>!MA@Tq?}KH!n*wgaI$(o6rqst>Gl_+O!xT7op+a1lj0CVMupzq& ze0DdGTKgh?b>5(!hsxr@9dObygK);G;syh|cMtmA``mwd*oVzI`|R`X_5Rj+#{rzg!@sNC zSlCUapvFM1X1c<67P~{$qtfwR#cN4fjxJX~<*rYK3?>}4$5hr=$R90xeC(7Ft?jh1 zRy>l-a#;l3qAoYtBqZd1L$*B9ik#Dw$O6^^PZ)&J@#@)KY9Kcat$rnI8&)IZe0 zqLTg)ysu)@EI|>|4~wU0Dy`OT*j(3}>+=@TANDu#B_mXsM1)msnHZ-t-(uc97d6M3O-aJ7}ATH#g)u1Q0xsa58SR5;BVRJF_ z7l~gZ@(_DQh+D+ggI!z)35X_MKD;+uhxjgp>|=v36pKt^TNKm;D!oKBNLT9AK?zDi zDBbOSn}HsDezy*Xk|5vxB^ok5LoW7NP4Z*pbxVCyr#mlM!$?`~)f63>M?V3oF;M@v zJY%5G{TFNpqbGAiJ$>h6+dcog_((7D5+~BqeX4C_63x?$L|VXk-jgVnxk}%ChYFMg;l({IDui$ z$(q$TpUG5-6(Fuft2P9mq>>ro)!H7o%$c zns0KB1lOr!8Ln;oxhhep>7+_2J`_Q`sXIYlk>Z$F7=J0wX!t_wzqm0?@nIA&&_;kY z=jJ8r%XOhX)r`R|oe;VR6QpAje!OC&vg8f|euicBf{Lid=fCBJI5PEk&y0RiN8HJh z{*_rlUh6-+#x};y0kvq;qn#>{Gx_9DK3grwg}c&A($Lz;bs1(v^pr&);Ffgm<)cl?75-?o6TQx}?ejWMJMpdwz z(E5b-b>fsQ5>x4x{_vs8N&B^T`S{Pe6>WXH0#zG&tP9`5MMrRSXP&3->Xxh@-1dVG zFpd5pZB~YCBvEG#O{Bg%jo_}}TqDPmA*KrSPRl47l+B3U8`21LcQKI1I|(r=O)^Bg zmt2;9!cTUUf(oQ!m#3vZ6fLacj}-|!FVTjwv3YNO zxZ^H)JAqa^EUl^OB`V&*zM+%r1)qWf9v5rD-K`Je?N3`JD_CZm_+!kf+VCVhT)ApB zT&4=RsA-s-IyE}&L(ykT;fb$Js-|)>h&W~e;dVG}_u%&Hc^a4iI$t+_fF`fwuAbuf^Ba*u|l( z29};3^8)W%2nAU`>J}d<`|@%ttvaR2rJZk$nmnS+yYgpv&7-g3Ux94vxIRDo;TXe*JhntFITW{_daX4tnl)PM^)%Z&oXl% zYfTXY>lm{-v&(!0zHlXx2A${`Iq&US^X?c|E><_x(_mL}{CRion_VuHMz(e?F?nV} zEka&{V`jAaz6+h0=Lq!Yr_Ta+s^q;^6#kY9{`YtN&G`)yyUc!YxBi<^pjNMP_hps6f9YClym-JO2Xxu9 zZ4fbK+ZTb0i+hW-Emsi+gDEJS{rvWkkd%HTOUS~=C{27BBo!4rS?J7MLM2QzVWMR0 z{tJ~M#EYfk2;Hs7b^|^PkfHJHb7viJk5K^w#Zt2aK-Asd{svZKCD8Z>30%#g<@Qhn zN+kjWbXanOUl;~qGiDV!S)f|lY|;_<_HmIzF;Hs1&Sc~Hm;Kf$fcu;P;#+nlFSO9M_$FCX-2w4eGngUeffyi}BtxgVVm=A|q z`ah*v4V#n%_@4pwQ$PZ!U0*@Ow@5pVqyfqR{FPf`PJlu5eSLPG!LN3{r|r-y-&bQ& zl?LVDpklkbJ44uDwhLg8D}a*%J<*=yF(LjuK?h*=+u^wpNkq}&S`;nK^%A5wF>ls3 zTVEcqXZEF285Oy>AD!xJsmi~LSHYRjeYNYUv#LD+AX+i6N`nr%8AJ-?eQWb?f*G(e z^X>7r)uO3$fq0rD;Y8ZWd3O$k5ou*b*e9&}0J5aIeBnJZ{L9W%ti%<+&mQM(ia0R*N9sy~AQT z&$V6vw|uI&!)v3zeGZ(1fDUXk_yZo>ZQv^T!+ib$_!1pyY3{LF>-mm;h~Z)SQF#Ii5Tg=g z6xRHx*H~Rn@W?O{>E~F&HD1L)Uz&5DEqqC%9YqC1U`67}xQm_u=UYx{2;_Z7Ljqp% zOFq`BotkeiipKh36leC>P6c4CHV*q+DEL2}oqnl~0e})21%-#(o^9o9` zvb!K+ELNJ9O5qTP6lOJ!Dt8GdU>C=P`==epy^lPE?Q7pn%w&HG`G2>wceOhR_$yr+ zXR0Hu9((l7p#e-(7~=$q0?oHa4oSERr>YG@IZ;tjpj{>@J#rl2&_!xdYf3W9ZwEyP z_RqSyEjCbSl`47lE&NFAPn6BvBPgN(UkRvTI`73R@>2ze4QmfQ*{fZ6y7{`lVwPVE zLiTvp&;p=$z;P0y8wUA`_&tEmg!~-xXB#5b`2oU|j4D&ORfd2~lWsJ{omxhBf4L~R z-YD5sfe|bR&K#J|-{laA^O5*y$I*r9BlCcZ2Z0XO>^fDeR2X)>HTRR%v%B@*HjpV< zAlrxvwNph$r&412S=a-HWd=9l5a&c`JkU#PUp1-+NZo!}jIj^a89SIY*SBzM;r{gm z8ki4>$W`naCPIZsH7r2(aqs838oK*62{>8eMy`H2C!(}E2c$s>I2#tOGJaTT7qap-u|h#lhqQ_)$o}tf zhCPAz_V>io@vVZb9^(K2zkMJJ)-4{bN8#N`whQ2>MU`nLd&s~+429m0JK{mZL>@-W zCVmB&xZM`8FESdWpG~V#F@2$(S}oyT%OqJaD5mr};p&9vPpTwSvZ^kx#1KY<-k+K7 zezJrj%Zcy4(#Pd~^}K)IRyTN9g3Nza&%2e1mSTK&x(a-f1SRXS2&o-i)#p(r%Rb3C zd3BL_DCPeL8#sKdf{K^YSmMB*Tx_b2*9fdDq%3BIP=!;`LKJ&&@7#u#;<^LF0w-$? zy_tU%OSwGW_!WC2r}r?)=$h*}n5dfMrjdWS6`oMs0?ahtQCYyJ_C?g4aJXSYe&iLQ zl~8#uy1yHtFy7GS+N?307d)O%b&ZF8c9pGwZ$>nWSp;NHBIsdKm%O$o22VN= z{qs9jYu6`+=n6QtffM2ZbUl1%j5{$m3>*!cg*DvvFHcoxIQ9E?EC;8bUD}`=&Y^!N zbIIVfzi;b`ABecyXB%(3^!cuvcSybj&o%0f4C3C1l5El2t1X`lpnIX{9;Kekn{?jW z_Rn&yzAI1Lt`#$U-#@={v~Ul)dj$Cm!uoB^lby^zrlUAisxG@$)N>ezF?Qb-*gCf+^2Uc6i$K8+x`Y&G4F~{KI>fPw-qk*n(z*=|HN9r{T@R zwGiPv>8s63$S46MH_7iv9QBb}IA#69gG+5{4+xq$J_B6wPsZFjMdAaS(9`$&`7-GV z2U(B~;w_FaNY8#RSv~C(B|YZ)LXfB&eL=q(>E3cHAL|*gja!Kt05Xu)rKJ04h$R8e z)z4StA29~d-*-q6`bim6kA{&w+j9))Nn`P57NZK2y3AW~1T)fjhwo0w7;XatiX~iU zaiZNS;PKdKp^yuRCwQGJ-}6FhkC7w54IkG(v2a(KriGVbmjBsCZ8`X!V>HIS2B-_9 z%pZH8H*(LNd8n_9m95(8-<&SSy~0|mVdoUBjxHs8-m#w8o2tYU4=7d%|&PZ>umPAowl`p{@EM#RE#1Uuxu~8%7~pEwfYoinT&q zOG``b!IJT4%ICyy?La+Q(S8|o4HeI1Rcszn)B-UHXO;8fRBw-FtH;TRT*lz-;Cr52 zuFHQ_I~sAHbWSCF!K&U;TBd!5*Zpw>f%o-N%fWFxH?d1n3nr~zxdKdN9a}PruryFg z-sib?hvUK5xcE)^m&YW-KSoSgh>M@NLQPY5^~@Hj(x>YO^gu!CAk!T3>qG%Yp%vB= zd;59ldC%FgZeRc-rBd0;7-Lg^&`|n%kmK{A@hVPgDJ8gHzvOCQVi~a9&+H58?_rN9 zb<5PpYk^rXig3A`gi^(Bp4-JZ=m)-Y;GgsFT*@NfGS6xArCdw*M$HWmGbRZQ#SGCP zB$BNRkN<^7=*S5x^vL6y#$EEk7m^Z~`C7}dpkr6sA^keg+#S6}0_ zU>SvAYrq$0;`?rg#63}6xhpS3CK9JLz%hQrnz))>HIxvSkEU3%9;6=xvbd23_c}il zvj&(sr!u#S??KP6252j@UU<5G)J@h8YJQNL-~>YN-%;N7%NmsD!9htt-#&nppc$6!l(E!FlN(9iqI zIy4I8EHO6r@At(WJF+!8tbs2N=ApbZz^PH_W_0tS*w2R|g#=Baoy@q0 z0q5Tq&7ecLhaZ68G$#CY=&_k#(Tz3xj{<1~;s(ujaIJ5|lSBA%*OTUj@d>N*xQL_s zBfvH|PwRQr#Sn38-C1xzDdT54s0sDr!g4ev7dwW}7wmmi=Ob5*NSk?Pz91Dm9@j}B z15xjtNuI4~N0=EEUel~Y3*|oQg|oMQsglWd(Ohb`mJ4~w5#}v1*c4;Zl@3%r4~z( z*JIZApqqTNbxB-nx*gsi`+%`+9zIi?RchjuShBPcg`1@rN*F#K3C_sd0gFYh0^NLi zY8?7cbp1TmNBgB1V`EDYGpEP&5=N52KFeQUCLr|ZBGGDoCE2u3b9BV<3EWE|=bSsK5s zWiDlwup;?#r!IxZ-hddOc}SQuJniRzB`{$5?XnxL72%?P@yj8qpFPbdwDIC7>#B(GoEHcHlMO?kq^DLNu~SKLcZN{?HVMvmM`s&yrkGd)5j1|&j> z4Lsj4buFG)WtGMN3dU!{lFpviI97Pi?$y(eRWh66kbY{h++#JaBT%#2X$^OkI?>uD zP{K>DU3%)2Kr>|)j9ABakl1E^s!=nauD9C<`)N+diIkEdzJDsW36JA=9LZ*f(WYU5 zOw^3m&q#H-78^Q19devl#_jF9Lc~sy^gYVdT+A%Yj}7Aa5mP@G?%>o8d>xw;$cQE| zwJ&gIWC}i6c0q=xyvnG`CBQvmp5PW;ahNE-{A9-_PK5YVb3@p&DsT6puQfP@o;P}h zlw0vNHeAvn8IQVYlD|6#&mkdQPnWoUKhggKL5!^pwKS!UZ3)-5PJz#d;a7*I%)+@= z`u_5?CW1(mak0pF{4|TC8^&Xf+UCVi9eXLuJ*ull;v$INHoUUZ2W5C$$gCG7M0=$b z)3nb)ity3mjeE%((KU4aLy(N5B=Typxxvz_!8S(X^+I|{Glz#~FVIAaS%RviQgj?J zoh?UM^K!9+I9*{ljX0vWDMUA81v3nn-BZ?8(APL|G+OQqjQntj8WR&ksW8I(%X!JC9WE! zcKRKpc=JxeWut3dTY<`e@XER_lYt@5z&>20+3kdtZ1h^R8D}7 zO?i0EjCCK~cjIJ1T-&)PYoo%u0+|#nOl50yQMCz9kRFSmpPsr-ZRZP}xuC`7NJQZ1 zk~@C*-~hu?Lg0~Ol)jg4{NsZQWE4FYsl|U!HU_0oQKdNyTkVXDVV`y_W+gLbf3Cq6 zMJSDyPivGy);czK*f@ndxx&2)E+dhnm{1(h=w#4WNRa3eo=ltxFzelzM=4}ksdCO@ z)i&1~-BQ+nHA7Ak++29(0aok7n!`-6Z+Xp)=TbI*f1na))bm#f1p9O{*v2YD&u9EE o>S6*Q^qms@>)c348TvR^H4M}nkhT&313uGTxc~qF literal 0 HcmV?d00001 diff --git a/docs/images/design4.png b/docs/images/design4.png new file mode 100644 index 0000000000000000000000000000000000000000..19df9bab9171d095ca3aaf144140f0d7dc8bcc85 GIT binary patch literal 27189 zcmXt=V|Zmtvw&kwY}>YN+qONiZJQHo$F{8rCbpeS?8)4H&UfyQWbd_}tf#BHYgHHC zO0=?~6apMB90&*qf{e7dDhLReHt>5f3>fe?Y6|%_@B{R_s+1^5-B0{;;03I+w9a=B z5KM}Hen4eZ$!|eGgh6D)Mby1Ouks;-@FiBpr8JvdJe6QbNJvCs24RtjrJX%JJrOgJ z5Ld;(@jdVduWzwcP(`=4IqSODx3kxOYdv+@`S@CL@#R8ViaojXF4yn+K3y2)zg%c& zC_%@@#)3aLz-U9zgJFWk4(wtC&+oSGw`~J2f+Pk=jZu_g)`YDK!2i2?)gvm3FzAd!E`|rcbsKIznJ$F%L z3EI6r?lMii0B$f&WKDcNIvbRK1o-lUSgF;^%+1A%9sT+Qc&^|&5O3-DuCz@9rC8%Z zfw?q0wcj*aqQ!&xOS%$@{x^%@ewg66{z^l=+JEJ5fVrXu(`$Dm z;BwqLnoM1-H`;A58736)xoTZC)X~uq?0tKCdpMb`HtG)me)zX{n1f$@zCEz9vTn56 zVZlT7!mE|bzx{dH&7{{Eap+3x&qH5|0J)%hLee65iTKH`SKeyzb|p;%%z zhr@2GqcQV`p+$h{Ea87rDr+EfUsRUhvRe2*^j{UbL%t*w*?M^LQhZ64qvTV+x2=7Yqtyt-^k0uhpltJTi#O=vKHnAq9z zJ|zf2=2^e2?^l$ZxI0V~`r&<&od=D8%kFw4U5~o9W+(f8M&UDUw_3N|Vv|~gykFOK z0njq?e<}<`B#nsvniKr6|K0l%aNWAf*!#E`5(1XR5?U#b|7R~0DXyly!G68z@7GKJ zQOHr1q0dJ8FIsv%vwsyVNTSdlB0=C;dlKx}L6qnrmZ@*TfOV9^ZY4_(jY^UHSwGQ`VoUpvtui#Li|6Sg8~byv}T`jJ4RFDAM7ejOtto0(XsmTyy-*~_}Tbd z^Np!OF<-Wj_jUlhYLrf+R+pokH=B}x*R66sm(#&#%_OoB?sb}}-;0MPRrBjq=&yCw5~n| zs6T%z3Ia^u=I51!{yZvEKQ2fSq3|IjgP{R{Teo3V;P0{Nz{MnhFwKcvZc?G0`tN!P zSHU3f!0?abf0wtxhmdY_QOxCxCn}_gudsg)__!WNR5jh~JpD0Ux8$(dh7*u7DhW z^=6xc4X1OvOe$r~r%T$H8fUM?bVevN66Hqk$4z&b03v5>-^-3~EEba@y7~KK-uIvT z9I%*(v)i+|T>XE)K6+|hfRn;A-N3Xbmfx&vjL`YVs01IuG#IGuSV{ZjN{HWcdc>q8 zQCLWmlOB*Y8KogqUU`^!!sD5&3}_X9|ztih!e&OfI6d^y<~G%R zesED}+vPEvfPF|4c^{#Gk<>fD<8vorpD)k%`G15W33`3L-cd@TqObS(Owm+$tuMCO zgPUx4#BtKHH>BIm|n<$m4pW=izj@fy6$?;6$nEQ9R(u9L3 zgcrL7p4Sdz=<0~uynnMydT%g*K5_c`Id zyQD{d4WL2yb40g3V5$a%l|YtP8V4MpEQw8wyfOUm)U_c1aA2PkIC|{sjYGU!Kw0Jz z6ws1hCzXq_zJY*+U0T1$V2 zcs+t7Nm^GIG%q+Qf1jDYHQ1E<_u+r!OCtg`oFv9FcYMy!bKJ*r&~9VrbDXL_sy}WC zzO5hKU2**1Bsjr(+9n*Afu@5&uPqN~2ga*huN#9KrrfhR)_@CI$qHjnM@=kq-|K5= zQNt8jG$`C)_~*SyxkKfUGE?6br=HVsqotEu`CvHgSowjfF{$(3?08u&z#*&a+A>?H z3f!IpXeI`Bk%Je^#*m2a(@%Xe$>sg%{oi)8k+xasL^ z_B>zeR5_Gfu%61de(&MNyqquc4}E`vAH`G^4I%<4M(?uzMS)4m&pe3%dUk?)122bj z!(8@j10lE1og7k{9xGNvNH|;`OHW+8hV6>IX;S}H^45Vu)aS}EWQ-5iqqra%v|7BLwlL)!UO=Rf|y)WxobIY}LP5pH z2zs!JT3hVmG!wAU)%DsR7R1g=L;D|2`PzlzZ$kV#-)+RpNZ8P`dTlnR2aR{#n;CO% z6C6X7G#e?X(Tj`d zOs703nYN(k!zWZ}Xd`Cxx;Y|Q-yex%R+-wwgfxh^xD-T!fTAbhqvqxlvi(5{am4ZeXbWdihAHbm ztueK#{dA9CsjZhG7S}wXS}@VqY(ySTpl`>QB_>0ki35Nc~JB$!)Lcd!_(E0YC^9lGN^ zG?E~yY%QS@Ts+b@?-li63t&$(PT6%iMu-Nj@OTP1XrW_GhDlmPmMp`W4r;gDz4O33 ze)byA7fPslP8 z(B;-l&nD7RqY`mSsOM}mYI0UuX{&@3#htD6FwgY}(1Y=@fysc8G%EpP8^~Q&OOL2Y z{D6p7sYF7IuStAN%UAu0AHFv>ed+Yh_cAg8{vbXUMcvN(qJm^YtHgA2MM_Ha*wpam zk|FD&na85Xaj|UDL|H%8f9QOq9)L6(2U{m`j0&p|rwVltM{T)j>vN7Q^_1nsvgNT7 zR&gUQOvi9{S`$|*Kg7CrgeyeTc>81G@uPD(VPIA7siDsllP4gmjoO>DOUS#zqdAzG zOBQMmxsoH(!ds%FX2*FSyt%iERP4ZOG_RuiYf-Y?-7SFNw)nid`5Mflla?>LM6u(6 zDT8p~Dx(O1rHq0;ThmnvNb_oRlTN{}s&8nQN+FpaEtn#d96aaJZs31(PPXzwe^bXw zf{%m)Uzad#%W$_{b?~&sO3kAhjOJ(vR-$h}C=!*V9KljCNKw$h1aWdm&*OkCRwS|U zO;aRizk_rEQQXcEEE&K}f%@}l=!dQv=SPQ&A&npUE zHF%AdG5U_yc|We2Pv;7ZOb|9P;!rk=N9_xvMvm~~Moe94(4R?#7yd%h90t3TCS$9{ z)bT8@H%ORSQzwY0aYjXm*ik3gA}J~k`Tgjlus0MaR-hhLsU2@6$BgE6WGc4T#1)5K zuKN3E(K8m+G;+L6aG9E7G>|sl)IY7MdN){fns-Dx zt_$6feUM{D1U0HynvP=%719zcS6RG>7SAC@O@;=ql!Pz}pDg`OQOH(jxf4;`O;}z+ zbUPXujo8(}15#i%5xkh)`HT}_ne(>=l!9@Vp+a~%9#rK|*>b)AOB1zhXY`9Sw!%?i}B5 zenGenGTvX}8EyK0ccXsPEDBdLlX2uYBtK&39JV}n{r&t^+)#{Pb(c}3@Ed`_b(-1- z(9f549m9(8j{h6awPXAO9pcB8qD201Y&!KxOAvbFYD*Uyk1Xf<_Vw`_t)VP~HOuo1 zGg$elrQ2|6 z@J&fDFktVdKoG%mFx`lY9TL1rFv1rO6Qmn{g-gd)^8q|qt%%wxqavcFbU7vE)u}VW zq-~UI%dgY&a5aK;E7inCWEm#8^D7RJ1M?(lRJPR&MOvgwDwh^8O$?M1Bax}9Af*{> z*3vK)Pz3UlH6mz9lg#I%V}jX86o#pD!e_tbKu^9l7cEux{gPb@7w~`w6)YG*s=w;~ z-Ipl%UTJ-h+a5{ib7qlg0M!T#r}@gim{L#36#Nbe3;u3w7oZ`PPIpO+T6 z!AXEjf{Xx_{L|g1u9uXJ@L!bu)%#1MOgX#n`YK<7Ky5;5;-kghIEu9=$-Z~cJ-|p z;8!c&=#+ISJ}IX$KeRao9rIKx*#WDZ9mWC+nf#-vg_Z=rpn%MdMG{tGps|=H${Ab4Q-!R3I zkf?y46?v|3(NlYap)OZ`wS;&xkx>0_C%GtOlDChKm#avTajt!u6ZnKgd?UlDlaIf5 zjnbxhxVW(Jn7uL$Jl+oT8rsKXnW&J$TDCsIB}LJ{mp-C(7K}LVj1}*FZ21Nf%nfBOMFx15{#XROg!gjF998q z;e`#zGzxeF<}4WAZ2;BkeXu8MhML^=!iv9zM7>u!OQMs_q}X9s1RCd!JLiA$6f3^z1FE~wMC zJL^Wo3sdK9gwoM;hOynyv|_svL9&9w5eTRM!4^hdJRtm2XCEy~P;45@gHvP+%>qL) zrd#!MEm64`L3Cf9_*Ye246HauUXxBOnuq~aMTf(NxR(lk)O?T5Bgrpq+m-UNi{5%GM6hoc~3`;18O zZpAlXR3ccM?Ta0MrW9-EtBq8Y_v2%8W`EE|fPB{y@O>UpdLMy}XElDmsII>acpU~u zKAy~`VOmJL)O$4CdB{e@mC_>4fog|U>FJe-8k;cDDaPa2xdXtIGYggToYfn ztMA49Ng@`JB|PCr*a;o%ywPIQzU{GT5=p@6yeCXb`pwYVI3nZr4VVJCrPJlgXyPv zxU6Ml0WVs;Qg1ZZQa?f)RHxU@7{z=uV6EEKzT*?Chpzp4e2E8dQpzH(U^%dEIy}NM zlB##35kLkyf66sNmWG!ALp-KOJ45!jqR**^QN)Esi)sKH!HorCAij&r-mFco;^=1xk|?1+nmu@k*}W#Vho54{m`= z2QW#b9d^L@^ESt>&imuKJzC65#qV5!Ya1dTdy@ZXPUx>Zwi)gH4BH%f;uOU?B>bz! z-JB|(aW8xJWA&WQh(K^*_(i=*=ZkxUL{Unv=^+viZY>(jKG7ze-j(p8-75K0ue2QQG=y5UY0Eg;(PFN( z;3S(!{*t(s^~a*(%-AljuwRwOWtHnnPDkp4RF@8-K|DuUhLa^Ewhr}STu-n*={`jbZxE2d-Q1Kun$&vrFE2pKX*zZ4Qc43k0s$WRqn-kRnx zEYRWZOT=QjWWHKa%gd!axI`0Krp(M|tCN|!$ozPjAWLL^^WF=FzUlk8@dJY`CeMsm zM{kHs6DuYj@p-yHQ9G_XBA4%B8J01~8&y%WOv&ga=&+zohjBo^kq;*iP6~Xy3beo$ zMLe~mCCdRiS6SJr#bhKgXVM4oNb_uSGPKu+yEhae#EK_Y+!Bc+{G^RV!!*Ial)sKb zT&+4x93U(R9sTm!g0n)qRh7Es$diogyQ)`UhHWcC`{?Ww(lRhmVq8c!mgt~U=Bt(o zCNJ_P%ITJ$xIaGAYjSO=yVVu&X-Sr#xX;PWg>oIkkp)6x5Qxs@PklJ2xnm@H1xt42 z9yx<&Nq=IJyen`GwoVT!7~BUsi1Ck+lgdyYnTPolC3}@aj@nl(R7W?lgu2VV3*VK9 zNRm5-$CU5xmhRbgA4E!vGP%2u1mh z!aQ>1MOtT>oB)=m9{Y7xD4G!BZsROZ%r7`&lI=+-oCr~PD{=4=Jb+~jslOU%YhV)0 z^DXzjK0dT4$AL^~Uy8+#z*Xpdr8}#gK*YLWLzIOv9X8t25+VbLr=+c6*#(n_v$$Q) zC@+^#l-Xs?1E$0($3D^v zMRn;hBw#QCJ={25#?AFeG*6rbMiOn@ezU#Kh|(^Cebp%s5oOpPM|dgd49XeRWD;@| zTqR95era2S(!@Ekg0aXXnye#!v~7(E6D?UbC}=r@z4EDzNUq#iJ|T%|&D=Vhi3Biy>ektsWTXxKYXVT@{k>rqIBJHja4OoT*ZO)FpO;W%AFCke@#B zuWFo6?7K53P-z${34L*+P*|8IyKohYd_|L6tNmWE^H%h+U9w_Q)X}nqP#9$H+PiF# zEEd_Cu0};M*zs1b_R^J)wl-Df#HS_QQCm+8@xoArvx&(fseMu&pg1~8V;AX%qj8s( zcqafih%1N-fWvRFi;+~FG%X7WDYj;{r}gT#jw(#i5^uFjyLcmF=KVeFEzHa@YrltzbTnMN;s4=St(-4g_e@Kzd1B=C` z`@kq}0uyL}*g3gw`#^I!*D>b62)h}h(>%9api$hG0UD+FS_bYI_JXIDj5?L3T`33! zZU_(GrBNC;+8pE{EM$Lh>bywXD)&15oEP$4;U1fSQ;DvQv}+ZV zXfn+9Z1R?kO{S(R;>YB9=e+28%tRpJ1*t@p=A~p2^pg=G7xQlVy@nL| z)`OFG5@wyt&d`9l$cm}nITR2U(cp@zsIN}$-!ZFxaDq(UIZ7sc zJyE%9K7SAq8A69w@kY&VV5_B+OvGDkZo0`~?cxs@nSm%-O=0)daDmWXB)f0K`Ektq z3~2Zq(}RW(a*4PSGgpAKlI|0s9eY-6i&n*{LkhIEl~8_Fktj^b*6^@Du)mwB0Ye?T zt-ji~@}wrzFwZu6>QbI+Vvo|jd*e3^JMF&dik-R7_uL5%cK8&xc(zK?%ora$Snl<} z4gJ-hM1D@`WcGoani|qtk6?atbodC7nq4?HHx7dA536eaNa@GAh{Nkf{r#Zl_YfT` zJuettPPRX#lGT1@5nRqBn1s+TNc#a$mjDOpw1I&zcuN>?)i4bl$v61})Y&C4wg`%y zZ4o$){>POMcglpbk3&2b0oPj|s5=@;y~3`GqJh7)Q+}TQN4Xxsl-U;+wuhd^Yzd<% zZU(5C_|&JKG=a0<@;j+L{7Yu4aWNejvTw0DjYQ1Du@RvZ^-8~vdsfF}q*UnIUqe9y zbn3P>@3^SYv71=@KJL$QYeoOqj=koybTW-)CNWgS;~p~#eRA{@vs;^7H9_gDa?#HG znvBCDH3=;cv(2FGcx*D6)XE)NWr*!tq4g7UnH9*cksFDf}5XS7Xn zu}hWyfym{8uRi>_v?L6|ST%ULQjUo1NE`jgLkkd>!uQIp99tNr@2Ar#A z=9Po#*}XgKt9HRlu9Jy0@Awv%sr_~KhAzWgks-=&xvm3u7-#mh;arIH@)ErxEHd)8 zBtE;Hk;$iA!pr!YdbUWmlY&3bAGg%-djts?>V!kl@I>mW9YdmYI^Rm677iR)*y z(-2B1vM5f%M8Ii&CCn_gU97_9&0j8+rAG-;U$i~vtM1{GqeiYW&LYqkGy~QhS5_~JG&<9rV#>6Djkr{y$H zMM4iapLH1XvW$A4Q#|s&^*)VVFL_|I&sT0Tw$0bfC{sH<)J(J~Om99H*QD@jNWgNA zqduTl*)=juAZ4D`=c3STgm|^(*mAz`gDPYZ;PD4RfG>Mh*-Qck?$_;d-1HY)R6u`K z2XH#{YlQ*^DK1F+kDIGZ*j_+Y+pilBVY0A(qiQ6VP0q!6Ib~~zk>unT=%eNxJ;5>w zS+F7_lEiQeW)C-byZ^K4A1}Y1xM>FV9&UHK%k?GqnEB6jq)}mh{hIGz0UQ($aUkT@ z@{D-Psp5sIsTeABtNuI?3_~SmF=#t3IV(=@f9C6QG}wKCi7(9XP^5_ZG}C5wax{Ki zl5p@O@;@600bA)Jm?g|cCCwqMxZQKiX^O&%9K&~GeIT5ItZ=;*80svMf})(u-Z?BB zQtZCyT`C^5k|bRyVs@RGH9*1Rv2n@WmL%exZ~*g^2-TWLj*oVM%b3YwX1v{;ITv_N zygvCLG|8Jqu(pmuYczQRY#r>QVYOeeU(BODy}%rLDp<&T3(|}ln4q!4YsKFFs~#^* zTB3xPiSH71HqJ4+{}#oHzS(0F6%AOaVtxo(W)V> z))keM2{CS~naY8q^=6OFZr$I45`H=xJaP7>pon5_9m`5&=$Kt&-Mm*uu>*=>!c&Z{ zRJOj}xkH-aqp_LPS`)j1OaWVr&K7QN9*vBqT%jV>6Bso1v96CRgiiYgrlLN3-!!T6C%EM~20$`fwlvA4J0 zX4~(9pZQCKU=jO+^G#)PZ4T2;n|0v1iL}Iw`D-=au^X+I>NT*7n3_Gg91g10nx7n0 zza;gj@q$a$N-Sd6x&x8HsubeCL?qP>TKAPJ4K_KeHg=2rH`OKlKDRxzgiMw9OUxoY zFfx`y9Kb=Jnq~N0-PUVeHPd~b*Gh&guc8q0J zlfGcc*Ku>cB=x@iKPpJ7tOJDgv~{F)|3^&?kSbH4Gz2%udgO9!{6|sHegjtU=)TER zkNiOVIO*6dsrNrfQP_e62%T7v*gXA5EefZL4?qpkt~;z)|HmOz(gVw?HT2}Q zg#Ujd<)48mx@U16{^bwo!P3Kn5W>>KOPBwL#n6LkpaKW3HgDG{`A^)!CS;&&O-U_2 z|BtT8k^%=#zv8rO`42l$$NWb&%oNSU|GyyvaTV#^zUv^7Jdwq6*na1}W$ZnU;D_ol z_E;5*#klr&;A`2{RrOCLO~JB3yT|owV+rp~(xaaG3 zBgj6fs6Rk0%HuqK5%mA4LLx2y;t zU~lv+=zaU**o}}MOV5iy&}*~*vUr{6yx#b|V)0^~#v^qOZnx8qM6 z1j1o5P`XQ5qt|^B6pFZSW=IpL%#5D%df5?=xANQ5_-~vuw}wmMVQPjNmC{Cu)NdeF zP`>0xBtN=f{o>fo<*>0P)t(6~zk&nTuzfdiLE8K?Y?rgvsDOaFo@#b#>g%joNiq=+ zk3MF(T8~PDD|NF;-0|$IldGhbqXcDKTXJ-Sr=i>?aPD`IwUbli&=%V^*sYzF-n|PY)R|Fk~s?vE>6GuI$hX! zcR95lyWP6k<*P10D%x~KLaSN~35qo|7|BdeIdC$peW`sVTxqR`Lli6ZK)kF`l3UsA zJVOaASmb*@pVwj3r#1^<$z)Bw6GDPRI-$nd`Q)VdG-NQKy4;Ax@%V(4D^-qNo7yyb z;iLi5&Kv@dar<6`*mH?b%ROJ;2gH$Nub1d#>E2y|22yD>Ms>NE@h<^v#WoD0$@0ECFx{o zIoO$`ahERJ%9QPDkXWeDiJv+`{p#?+WT6T~m>Ip>=uMa9Nk*N3m1vGho10}^Q)h6p zQ^x3hFOww6*-!xWA&zzKQ@~&)+XOhwGI*l($Wlvv7cHt0#Sm|XYd^Bz#*IQX>4B=I zG`!=1J3SXOb5Ym%lc_kS}gDc5#txp{8=uP=+MoiHoAa z)}h`Z%OijfBa`5V;_$e$-<_u9prq3LHa4_i-N2oZ+TNAJ#*ssvYaL6$$(`X~0i>FmA0`&MH(j!IlLZ-}n-c=|*^ICOppb=DP zhb}MZ6>Av}u*QXjKrC=o0|w&=iBDaYfc$AOLOD-(VDn#${?bm1~7Lg}B8|-&F-P zz}T%RNJuH@W2zVnBc$nTjNKGs+6jukODAGS;SX!Cq^{w?LJW`cBGZO$Ka*arDNXv< zi5Hbif$bldT7rkD9MBw)(g8yVx5qWD!iQ0*^T)TL zbF1zOPGx}$aG^m;u}G8g0-|F8l2^M0P?0$cQ)UdXhzhfViR65qAEKD`L~F&ny0A%~_Hv3`iEsb5UqK zGKb^x$L6e%WT=Z`Oi51i342YPgghWW;P-MVY(Y)dkkG0%nIEPpn>z>lgKSGOPCxbD zCiZ|=jsfL;g7GC=EkFwi+BCrTyXPU-sc(N~jUZ6&4Y-nl53^j+MCq!dM#V=!DjSd0 z97eDpDcCm z8!jkwr01O}-_hc@v+bn;7uSttlk9u3 zG#>HV!cy8bZ%_f)YBKB`+50h>4Qs? zF^1iaO`{vopkTtLP~FKvXUoQu;b{&rvTbd`t;VacWNf;LWHe#TvP#3MQrccg7Nxs_ zsC_H3EHjHqR8y0?PP6wQpb3}Oxdd{xrB%B=W6wMpMnjpj}){BH7B9SRKC#(?2Hx@r{2K!}pQR(E*z$SYcMks2C4^pk!GWKLMDic|e^c)( z@K|JsN5nUNZ&yj{1J8UMlQ_NU3y6eKjiIrgGgzv;+Z8=07RWC1$&|mJr$RvgUVIY>9dP=+I&(ff{oW-K048Fg#l}!t zZI^k0nYVD5CSaL(6qOLsYl0?vTu}S;XFkwGxI?KiJKypz*BRt$>zxN9x0m2|<(Lc? z$XFH`B)SIRr%=ZbsiPmiw7e**Fze6EWLf^odnLF=5kQZjh{Czr$^Z>Ax` z7lEW!r=1I@&)95rpbONsx_op}h{I9fh8t3mqrsc%CbpnRmc{exJ0J=umX?BN9V~JV z$bj6E&^R!uR5eBHiI8nEW2%$`Ec zFj)~h72GQoRM4$GAh_gMT)5Tqc|U!*TphVOn$Ox7_~&UA2u24-n(JHuQFO~D7a&wn z5cuT{YzGniycPR&0QUJH+wABw?EcenF-l+B)ZDDZe>WnAB!GSZP2@M_@l?HshS)DNBv0FO>kH?<{LO{l9x8ed=en$JvAc#pCfajDbz`geM z=cD8l*prdVW`&qnivO}37zliW#d%H}&Blk!aKmvRK7jQT4x7c1+tTkO-ozvEAytjJ zoeD#1j@{S51GlZZuFE!2;JM>zMKAd9pW<)imLpIDfmj3gE2016LibgZ9Pm0@lY;)~ zY8@?6F_U<+NDP|SeKZDR4{`SSN|&*b)V7)7gl=d` z+CP#f1-7zw)+#=W1ktHJV4hlE1&b;PNW_V~9<>eUYG|2g33sbRKSZwqEuqU#r27pt zz8VDZALygxh{}($C}`!jpz;rLWii_viTF`*%6P)5W~H868ks4UxSVCeC_3?|4jTk? zxG>7C>>?Inx}ld*t+b;WU^4F4qT&lx$;*Cs@H5T#^`uy z0*txnh4YZc77gKb#0R=#tXr|HG;%S&9mm<*1~1wS@NQb<;)x+h(D4eMZr#iEW_s)0c8-Wx(kI8;fN7(E9MV-{GnK0mQ7lAwFsSQ%CALAZRuRa z9oo{%Js}C|XF;z^?qcw5xl-}m9*%` z&*&{)f=x0@Cu~Ri#d)72X5CE5_8rI~PQ)AkFlR)U3M7!4Zy>W3?2RYTsc@!Z^ zWfnCDX-KSR+@18O-suEgnQVWX_}!(9ZUuPQccbpQa(n?X8R4w`h4yR+()vkWUPgFDwIN~H`PwO8G! zGA4R>MBM>Y^$r&!_cX~jLIfCvSPLpKmawRCu$%RpIwLJyoq0MkN0@GYmmKpY@@Id0 z+0|G4XxE2Hf^D(g{y7~}Fw*S(o#wZ})R8TDVBBwtx`w(iHefrG=;can_8(Hj{YVos z7Ggc)0^+Z1w&du6Uu_P0kExskK^0Guls(jpSiDXLgzif!rKM>yLX}Yi>)oDCjQ_wd zEB!EG_Ru67`*S2Ip$BF~$S{q0TaW8j1qyM_ONh<*5Ze_$d}1(^Q;R!17hjxGgeQcd zCMK#U`e=|B&lG>Mhrp`(gggmRvQ1uFGpAdQd=KSL=|g$w~IOuQo=Z-DTC`QE*bxxQh>3_uiD1z>+I?CDUPS@7_DUDELklsOoN`**M*{nAfPSaoOr_YWW z7h$d;WndF<=Hpz?_*fKiQKzN)uJK+}!M?1cF{M!4?HWcaFb}CvRC-URrMX_Xj(u|} z5^=C{qr~u`1X_?3{MKM5?ho86AA-K>S8G!Pz0g{fD=iCQ^;mC+dFC#m2FUg zP7HppCXAwH8=3O)?+7HI5mT2vh^VrGCHT;w8Y6{2L?=tF(d~rc*)Fs+j8`o>A9GbD z;~8>c2{k@sG0^$&{D7G_>e(Mw+dt4oNP26*lKkt6g%ZUrTrZTS zcffDtb57B=6GtTU}s48yYYw$4y$woJv&kN%L*i~N3n6L!EpKqP~; zNCcqBkM5)NW#Xr^vrk8Gk6e3f*b`a#QIA@Vp_u#Ep zbO>M4Y#vXlKi5z*kY;sb)j&9t1hjxD&HaN_J+e0I;_6u}JYD6fmLJ63Cpqc}nb4}* z6+_UChgVfe&26+OhA^The!(1Ja^-7x649TrL1xtv(%Y{4Wx@&}g5ki)3$5qzqm~5RVHBGZC8^;^ zLx`oDHYE`WEeQ$_kYOWS$jjPWs(jdLFK80Il!lV?a~#Mk z8j^z`AT&S!`CkCWDsl9wJgWO7ztei---#LmZc)S531l$6fSrHWz)oCPu4)jH9W+BT z5hLU~R2nu;Q1p8rVoT-WSh5khtWFB6;Y4ox;t}h^lv(fXAvq@z*4ahRyI9rRcUm9Cfye zFddOMU7%bscO#e+YSvn9TY4Y14OZ zQxn5cLDtHW*nE9Y=Tn%^iy>;Sdrw$p;b5~5A#X1{YbKJ3QU_cts|?qQvh%$Qhzm*R zkxu0olEE3EXwdNEqrcA0(@tuB>3t zHw%#j*yeyg?=I&L_WH7hgZAXp8g7i2fnFl(V?!H+AJ=#7$k9!}Fe6nQjnid%ZSOlk ze+oP_{MCG=|Nq)L%ebhvwv9^%R+O-gmaI%zmm|1yWay1{fw#=+1MN6OGwEmETb2 zlaC4ijcJVdMe3qDyc_BD=gcp1F=EkZY@}JSFl%-*;M({ZpI&Z+ zC1pd0uu=Kf-JhDvqdQ2kb}6Y{w%sUT+RM{wqDF^tr*i!XjKz02)Obpw(T70Ftv#J3 zHp%A0_`67N8}61fvbihRLRx8@oVQ*zrI1TqTXbt}a^{lY8$F}p=#kbgBdnD1RgvuQ z0<}q~qQhRAG*5pS9&Z>jZdHU!n(QkEmxXr`pCB~#m9w%%O|DD5Px+rg$qu4b*$b64 zR)^4|d3CG!?N057HIb=usnkSKa!|OF%6!cG5TXg?Ci2Zy1_UEFoaNt2@{T-3j3ZHW z)GTS9&TiE3zWMa5NDteUp*S_%CJ!|QQ@-WeeyLTmG)`BeS4mCo=d1Dbrzmi<^Ey7J z5uw_>I@ht&uerM3mm=s)kBNHNu@yFNrotnNc`@Z~m&2s-`4#YXHS?X4FSo4eKtZ|*DV@)(_@i;M&;Ne*_4K3P{ zAwXVWj)5Rz6Z$_0d!aw5B360evIQ?_ISZ(z?6gmkk&wht1C~21Jcwv2TagfVA9>7I zB9b~`V`K3p;Up8Ou=5(TBC2K3nv=RG2OeotwD594TD-K$8jPj3Dz1f)tC)jW2It>5Fd^vkQuq%i1{dJ>wAtv5IJhy~D)98tvpxFW zpW0Pl9wIUY<*u|~ZkI6V=quAKYkDGhn-+^qKJ#Um@(ZP)CxRQ=-f?t}4~f89*V$K_ znd-m_@#@V_MlVK=J$WaepGtf)1;TBQOPC(7$n18W9P3HOL8*BB9BNG>g{ES?M2~Vd z)}_KS6UAf!E(&^7Rls;Nn0DU_PP}5i2dW+7H{an9Zf@K+n(rF};! ztw`@yOzKa6u4mmdqT;c-H{;h!qs_C=o#RDlk*$EKOM!2Q(ABB3yq8gBZ87HW>Rsg7 zdEnSr-H49xjGlFGy25a(Ps%0`org?BI8-tiI5b$68~E#v`)6LdCokueEFs!`oLtA} zmVEny5?BG44H{@GZEx2edyU#olvef>5U<;&*Y$_Khj-lVW2r(CEwx_sk*=^`y881* znX>UM-MxZDB4sE6%blD0XJoE-ms1d0=e~XC(Jcou=#5kNKvkgBzQu(Qvz~X9iaoU^ z?B#2EoNEs$kF&~&z&CSadrnq$HxqU=N}d0B0tyA3ma{v;P(LpH&0FP5 z@j^pD?x23!eub4-f#pJEfAGei2j&+H%mW;=8%TrNH-NRc7 zl+q%-xsh)P>S^xGi6a?$e-csm7v0jU+>zR4h)6{vP`tG=x{qxwa4KbAK)^R6Wssmn+9*=m(8{%{$Fs88T%K zGfp?l!G!cn_nIVBV*L=l_3eqkmwID6g1%Lb)@kPByZx~1XWiG)3tku7y?b$9)HF3# zvulpVt{3USE5z#fC{5qSlhS?^R2)p3N{=mb7htFlH}sZbSD<<^Tn-!6)q@^7wjZr} zj%F3{hr-e4nLxv8-Z$GU4&$4@5)owEJoa?QSMx{)H0licSnU5aG7)r>$WHqG_Ra zg^!k-O~N>utxmJW%U>ijZ{~_WS7Y|}f&zvWv)zEW5_ztYncaM{L^804!MVfen2pCyIw zSw#QjJsBbz`bWd}n_sk8hWL1Hw&yhhZ8|A=tX4AI=F9(Jl4{Yyd(QCwsQSPkA*2`4 zVl!;ecXw*hk+*9*TkGs&U8z};kf&K?a1%K{-r#gQhgYwssumoSw>BGJ*WpWC4HQrS|rjB=AIf;=gRcna13(3;|5k19G(UtqoV z{$r45RU_|)<(Dm6`?4C9mlWcTrSt4x&VsyD=2?03bdkCs(GN52rv zCQ@z@?AW*pRsXV|{MxbMTUgDX{14=C>kXV4$45(-b(?<~Mvp*Cd$Iq43H1-IX%;vt zuIdkcmOTF3#&28%;qPaU>!pDuqy6pWQr}{PA9s~+hi8A*G?5V} z6XW(pr3%04Q|+gCogfMpme^xnOPL6-&%Q7z7B$cH)m0; zzs~_<`MQRm3ZsU($H1}}SUmx=d37P?yTe?}EKP-{JPcm3hX!FTsDG(I@G zZfPK@7gpug;n#G^|BL(i`EVTEsGW(9dwb$fol)S|nz7~NhIR;xG@kLSguPx{0qX z)s(haKHObz6LVRZSy~>|jER5_tCt$DLhV0SzBTO8;R%5Dn9K&l-+`XZjkckudArY_ z$)1+55i=rl%wwGrUC9@+yZ z>-`Vh7!1Q92WY25PY8C`>t;7Q7<3b)!L|Ssh7jsSSqy%@PoNyXu2(bg&u4gU5uaBW zG+)5bS+78dEXMg&_CIDKSrkNw521nU1ZYvYgJx|T!2LXkz7SGELl0_ITJn*SBup?U z(l1?~^i!Vf;nFKU|5-b=HJYl;q4^nfdZY8~$fiKgCBl!_7Kd zFjgE;!AHxQjU;iT@;Pi#u0Jlm5e!+!abGY`j3gJN=e-cUza9hq6JFPyac~U`8Eh?q zbovcQ{5*s!osoC;=PFQaxZZ?G=tI*nm11OXvsMH6MMX2psUb6^y6UcL4%msCsEk`N zFvaPtcF-PQa{%2@4ILe;0}hfEzfRlvzDM?1xUXMI8S*JeIxM$b1DS=k+Dt|A1~{%2 zSf}pOgs|WR##*b=8<5Omu$9to>Krs^v6;kihaSS);SWK>qRrn&?J{Y4iVlxSxOc$T zC~qk6_Oit&eb6H*su< z_CFkVqV$x3`E5n2QSu&up^m53l~*s1V;>@}Lr$rn15K+S1;|PjM7~Gn<);8_)-AQp z;M91wJ!a_?7M8E{`R1$O(F%XF`%$DQA*I6N85nui&QyDIsSKhxMmE9uOJMM$##V2B znaayk29$y&a49;lau97humZ=r5SB@O$51k2q}21KEYKFk5U_7pMZP%*z*TzmTm^K# zw|Ac0L>YetT`lQ;(9K>xM^TRhtg{hNi%KRddaiW%oui$VC?qg9p7fFK`X+N(4ox#o zRe~N01MHmGTiGb5?^$^RjD+&t4)snrbLIm*ZiM<9z-}2T$_enZ<5+M{xoF70|8+WF z4Q{H+b;u{;<0u1O_k)JI&xQy-pc{7~YfF!=ED3QlxU!G7CiAXcjEjtF-778uj5tw! zN_m~j)zhE#Kk~yFgh00hT&f> zfC{+o7}*#TB5Aq|zA->2VD+0z zC=eu$CJUH9p(9bNLCG^_rozIsEVOUTHmX7IXpnA!yH(0iIh>Z0BZcaU17gCAKu$4f zGg$iL{n^XYo$e>1WYBI(fnOjkOYgr?t(+kOGR(=0W)~}rzSvz7JCq+ z)b+T%@tgoMPy2DC^f86d)uaL&Ij5QIB$8A`2+KdK7C%u+IIRy54Rb}(JAheGR3+(# z7?p}e?^W%mi-lBh)elT0=h-gRvEVw8nFTO&aO#|tQKGgxrOU341#{l6iEnh`V)-ba~8&UKW z#hGGMxksMH{=(WlruoAe9&4dbxiL=adJ}-7+OHh}zCo)CLr-bmgOL=+DGE_Smn3)O zQ2zoWW)(Ym0O_PkvCw%%w$~DdlX&bRx7VknB=nR7Vtb`{A=lyS%sW{VibtVUEsJB8 zXaWo{qY^abINA=kysU|+qGr1G0hAevK|{3+tSOr8J6G>}m}n_R_HiH!?n6)Uiyk|4 zR*i{9vv@NZCGLqA%a|~zg{u#o#M)3fnTf47_YS}d(4bR(_Csbeur7W-(xzZ*#KKp_ zN~ssfR<$=+@cL?_hE094BeuskL^_YwJV_>!tn)>8jX7fx0Z|UesjTMEag8=9!oY7# zx*2$ff#ax8#`485!@E@)AJ3K5{#l3l6}rW0DI*54M>0cXMBBM`#vxJlitnC9%cciE zp}ju)4lV2XaTBuHLW{B9HoyH^$!FeZ%ljm(@#J*wRK~H-rQx*ShFBLOiV!=A*jk}S zl0JFi5>q#ftkJD|2hYn+^UCKA&GFQ;olYUa``2i_?4x^N=mWizFZ{ba5a z_udFbq6`5=pH(cwy^`@wxb6?1dv$B{DswCJK-inP(~m-aV{KPK-cwa;(->5~5u~En ze`us5mt(X)xr*e5FE&c$wcro6j}dcOYL}uiX5kMfR4|V%N-y-yHxu}AOFoj>`;{kx z?#;G$eR#uqLL}#%dGy6QLt`QFwpn3@fs3nOwQljeMK4+`DV^ndz0{<&#v&qxl-E8@ z#MF)d4fQ8U(_VnjecfO+*z}8Z6vvvsm#d4ov9V-cT0=$Wxyx)C8d8@^+2WULoYez< zm#Pd74gL}W-h;!h zpx|Qu*Qr0x3eVI$@YCkoHBkr@a%3JF6DVKqa_r0pd2bNpwVclPLM?S;i!EWrok4_5dG zG;*fegAG~Ds8;^J06{U-$TF?iW=yyFzl_2E_YnkR&7b^l>i?+%Dj^zZ4;cfraIMh) z-q3sqHxvt7{;d03yUUA%Vq4yC;^}|Id~p~k05!jCm;T3bR$+R`KQOVsqWfD>t7tw1 zFN!bHzAXK%z=5R^1TT=JOM3pkQT+7bi)Yen-2Mw*;gVr)xv2TejznA|@qg!oA<0abVKwJVSucb4uP$<;+UHw5 z)vw#_Q1$IZfur5TZoN%}caYc^=FDgap3WpDX*l(|vil zGtvA*?j6LzKD`b3(I?CO3)oVy+y&DElzs zaw6Mn5&LiRTwew1rAJAaK4gi|`0MWh&!OE^o$y+6VBk`HH7yFYW8XRPxRU7Efe~Ze zDd8R$XT9w*3@#XxNT)_RBJ4e|QPE>XQi`^De$?WznH^zfX0$eHEQk56H2vD2E7c{9 z$Ri6K*7W-;LHIpX{a$Ec9|db2+;1~;hWpDJe15+@sLT@m?D%b>?A4d2?QyGQ9N8!S zmp_(DwkZnIU0bO0H9RhM`v=j-p8qtXXHdt%+p{PHWodZqmW1!!jq4TBYj$PWuv(>1!HTqOdYGv6^vNWZd*QP*42*ws81<)S#^Lq1 zL{yuD^p*dkuWn#L2RO_&zzquwa&9SZn!Q1BkN{9fwgKS*5TaA!;{yqS+M);0-40E= z`37zadAD`|_igYCptZpqtfJ!Bi?6K@_&eX5-Y!*}qoV<+bwqkUg;yyEu4`!cd^UGs z)o(ZkKHkO^7&>tI4R6)D|8sYW2~Zt_BH+W-YB1Q@GiGM}b{}Dpi}L0}C^X;S4sbj@ zPCx;tl=d9^1n35z-4?C!my5JH5Zok-C8=m#S#&A^uz~_`m06tLfW9Aa3xDO`PCElG zoA~zNL+}dtxi$eYn;x5gC)0m#t|H3&0iS1@0l?hp&cNgioZD_aRu#-%G!OCtlX`v{ zz$5cwy^F3g2lpG5Ft!J|th}Fab&hVBL(xYa7Z@QKQN!rw-SJc|7>J zu|z@$&6?QSZy$`_RvHUxyDs&Tz!M+=>{lV7549M;TLDITLr#>|uxV~MDEsNV-tLTx zTfDUXU}$Iv2Ij1}GCjg{FvBqG)rG&$xIF^pWFRK8TO<|)T#Qqh;{Gol^Y;kq$|Dc5 zNK1naV&y~o*{Pm;UD#TvECHBz-^OzL{U3+(Zhtzc{n$Sv5tSXbPLt~{z;mB~y^%Zh zV4Ki}_5(Mf`?RV|m9Slg_XT#ghiXgqm1fAu497x3Muu_=z_Dt9V)G^GU?0^QuhVkO z6RZfwTn~#?YGW}hCL9%gx>bUsu-EYmB5G>Irtj*2+g+N$cE^ma1k|b7v70zs7b-B( zU>FzQNG=p8B6pzvp*>r8J7>VwcD44Vyb-zI-ea;6T+#DSKfdU*>OfLqZ3lHS5MKPQ z5aWOa#{-bAcTus7t-5_a0sCeKzjD_qKsSfLR=$q=TF;bJw2?fxM8&lIM0Q1UsC(F2 zjwO73fG0EvgIMAy+XXPfnl=;kIxfLkyWdQFU1$XuL*wEXpnO&q2v#z9Sn55XP9B`5 zmUUG>2p;Y+dGSZ?o?stq%B=_s@)IrUPpJbAfKss`*EZ`7FgW1Pkw16LOjKGFBj`=R zmXyADB7BbZ^(8^l*XQfuQ_7;`BN^{c<@9&5r9KXjLBfErbP43OC%3eXbD+dk+UQ1! z;eSvPfHjYT--!tR$M2^LRUeQZtnQ~9gS5Pi{O98@P`f=$U2&Qs z`L~E99Tx!{oJ}}VsX7Pzsfc?m9iEX~U*W8VJ??2E;EM%SLcr zLPzc|Kis1{X^Tc;>cU@QlAU&JL1gG6N>7-fGg%E~NFPumv)%;xiK!Y(Wg!N?VEr>- zH|{H1lR{0-cY9}dcYCd`<^i;Vba3Z_D=6x@%=~Wql7ZGtaXME}$= zDxr|$DA$Aes@thR*CPRD^b;F@>~o)qImY%6?~55EMzu|D*l$Ainj&uy$ZS7SZ0Icf~7`2 z@!LTZQt|FLsX9qS8$P!uGJW!h)=3i>FQ8F#w0aNgD>p2-7BetD0IS7?2YSrA8G-f1m-fDRUP&xzxWSOAgi2zq#yb6hspnkl8 z(S`UITakFEZHiRvjYASSt5rztZ> z*;rZD!!j4p2+@a$&_n~O9ucpF0V9|GsR-g$c}2$|kwu@U`!SGW%tsUO5;#A`iST;; z`+Sf|G&q7)$Dq<|&obzj6>=^UT7JsI6EOJlx$AbDuA&B`e#gqA;s}?b!+9?8SU0R? zus;Ev{%roh_-bN|mBB?<8Zad70cOR_XTH1hoM|?DYQNi?#OG$0A>RcvdZ&ZTq1Q)a znDLqmosLt2ifW6YD?(hX`w+h=_?IZVxeUg{W2**pWD*lTS2r)tP(|g;m)RsVBm}C- zTILhcV$a0muu-Dp<+S_6ztPoIEe~6D33`Y##1`&gZ*D1KZGKYVHo{-Zm`5`w0xge+ z9$GFneCbBj$tPgzkhq9kJMnk}iwJI!VPxZ~^zAQ4zp#vPuEh5j8*sxXfff9+V7iif zyfYHb88N6N?Y909U@)_2kPK0@fDZh2Xj$q=(AZ&LS8AnLDp9``{Yr53Qph6qvGiI~ zC=5Kkz^{wtDi#^y=A8C+R<{MaBOh~7Dna5c{@vR?(e^ocLBe>*gP{o74q`WLzWiB*}Dt zV#lV4NDu&gB=14ez>Uma`bgg;S%(TO%xI0}ZqgLj?kAJN;1&3^o%>@cA(!)x}x?p{x4!ziK)L6iF|47 zZ|;A77k><6#|ku#L!uC)a=yn6>qV@*`4oWIl^yw=`?g%tTJTQ@abe)>Osk#291=2n z_EP(>t}V;(v+5xr}^3KDK*(J$H#AxTMsmEXOC)Bt{71VIA-29jF;{Q$yQSwiGp z%_PAQPxeDF{z6K8_k$1;`VTb9 z`T04Kjib9ew)+xER3~)nuk&+s3ZX9~QLpyp8;?xSffh|FeE!O!)XI#sRU~9CR zL6r~#Mb~jAJ}lBB9X2+Mq}KNHPMqMQU5iqj6t!h~8ycBx0^;_s-cS#uqN&>M*T*Gz zJKQ3PCOVNXxO2_ct#p-&a%uxGgx$|qGIDC|W^`+=y9u4aKC`pukGE<1uNFMEJ75bU zecOeU(k!#eQVem)B+&EaP#%t+&q{u6mldlxv?>c^>KARBmYMRRSmgoNubY*=uLEgm zI*-4PtMw-q$FslfA6Ln9m|CQ3&4vK2_XLBZ%X7c>J5zd{3oUY^+ImSzJfc0Hc760g z8#4^Oh(Jh<*lc&`m%S#F>j`$R%UHa%fLf~^3-zZs?R8!V-KYX+wEd zFl;%9V!sCl&fnp1{62uB*Xww~Lgs&iqsM_970k6BFBbUU2=Nlh9}JJboDxOhzqwnp zUs53kYyZ2p+zh(w@wjTuuG98#AnTC~ zp+>c_f{5g5(O(On^*1bvV*DPD$`(ykzE@aBgCJI#t(VXw2De#^m!74y6euajzRyY@Z$+HX$4(DMhnkl*`G$8E(lP0#Bn`)UA3-{+*dRbA5* z3k?<#*I`zQT8?WgSR~K;oZD-1NzdCh@I94=(+K5fUDt)gtS)`OhrR1b-rqpw@SMk4 zfG?cwrE1sUZtZ8;=UNm-?B}>|>crze)Br<{dq3S6c*CjO&-%X2%Ih%eMG`2Guj|fJ zIYC^0Kt-x*Iz|_Zni^I;c4CHzyiYN=<`e|KKb?0j>AG+5p119U;@d6Tb)EsY?lM|+ zfP*we^|;>(i{S}6j%J(m0B&7f@oCv02uocf;2;D?&*P`>pG!@%4Abb5ZTP7ZhsyyR zU`{?cajL)f+>F(UWbvHjKKS+eed&EPn%B=O-n%cem} zn$_W#P~X!CbsjiA>SIefyh+F?0@F8xPX6yH#UmKT1Cfw?34d6OmIzUzUi z^||AuM%${Q1RQ@*oFv}?yzuSW+UF}i>vVl!Y@`z8iRu#py-Ag`%WX1@$`*3GB|~5~ z<{m+r%60O!(Fq;n$JH2<{|^`(S>CfIFI~HP{Gu|@@KaqMGVfu2P|Jp!wh!?i1+(>D z76y<;QjW)VggD74%kv<Rp*BXe}ZqK@-KNImlF$w+$%i+z41Q{YkG7qA&^;Fi^> z$5BbzH#Fx-?y@Z_K5Z}fm#r}3Xg8JgPluhqU%lIzsam>xeg(jp<@w$LJ)&c}sAXL( z90Xgke(y~@r)~K}4Ri+8)}bm0_uWZtcNcg2=UP2rx^>^CCIrpm!TcE3^w z;WnQU!x4)l!%#)$-i`PE{k=6EmHhPE^`r%pXD%=qhkuVTbtf;H#5$q~K3|yG3)7ov zc(!iKVq`n<%nNh~TH``*~h*Kb8rK2Gu0lU zQLf*QZk@1opKm^ntD5I!;dphcwLgwGrr_S*z z6>~Uojv_#ZG|%zg5)LD}z2p7eOh}@tUkoG8(hedY@)}DX9+IV*x7b~5GzrypTcI3c zCqXNi$@N*(w&y43n_A82~l1IzVgTEtvT8~GTKvgj93mN}3fT>cOO(**v}EQ8E%iyBniu9qFoe9@1yt4aHEmrWfWe^f}pR7uyhxXO*N)T9ThF^SV$shFtJ9=fe_Vp zQqw8$u$x4%RT#xc=7j6#6HYddKIn;@f~}eDG)lWt_CkoYOn$@TS!D~KI-ffm+OYzx zRiPgO?B-TE2QD$7W^z&~3dqI0DeT7JY?_A2q6|un2p-8!%ljg8<=QQ|7Uc4<$kSbh zD`?@YF*-xd^4OWeOI_$mytHzxsy}9qP63f;fS{KUgh&oK$c%ou_(QZR35y_w3^ko(H@wmNlzM0CY4btTvr}Rp5C=>K$ z@jX81qxeRPRr(S>u&0RLyAwfM;ElTmL(er>E~+QY{C#Ck@Rv}*GsXbp}beb2oVY@bB!-I=!x*Y@gJ+0#hoA9#1P&V7lRp+~Di6H(tn_gX3xOv6!i zx7rgUsg&?)FP6;YZqZA*P@BFAiBL*?mH4VR0r`BX(#N;=Yhi-PP3kH!kHQg)zQ)lh zxJleBsE;BbI=z!ZEPp^wc7&LgGgEYujvzot1%xZ8c_rg`kJ~peVJ4894aJc_$(Dt~ zAl6$HM0T;4l4siNJ7gTk@0MlTQeRwjY*1`)H9FELiarECp}0F!oHBxyZkB1upfIOk zZ<ML)5D0i$VDXNZZJLs>Iv9@$ zj-_<17-cO*agAjqEo=d^DPkSl zV{uyI#J(;}R#I6I3F>gqu0npg&g?3oHjR}-3YX3_VS4pXiF_%stQZ(LJBNI)bfq9K zYC@_$z7LMT_H_JNcbsB9KxhE1Cs0+qZDO)D=3PS(S<6p9Z&98!G3XnO-Hd1(_}L;> zQB*+*N#wre?bX-3ifYMV*hWt}BzylL7#W9?aY0%-ja#;M)+R);CutkKaIa)IF^zN` z=E=HeBeUr5Huo9!aa`M=EFW&X?=Ty%zOYe94f!#R>`|F_1BPu z+PqwJbrR)wpy!gl_!6WAIVtp6mQ7Ze3+jh5hO&u^unGw7DI$qS6Fw*ELc#3m?{R|V;dudTwMmekz6r2PN&#nN!QfxZdadm z90$O`sU4}f?wg+G@}@AtwBY@*@F~9Q@y80udam(EDzSVnvbZuRd~S@;Zie|JUx=t& z)oX%LLsI9%f-vm8B_oSAsDKvH9#qZ#N} zB^;0`H^Ax&;T!6#j)XbX#@u}%kGd_g2(d_|PSZm+SWjEmn#X{!JARnVppbAj4wY$A zEHm09DHZB0L|sQOLQ_StO`D)UG|izoCMjQzBMhU+rQGDkIa8-D)~ZDs$d{r<&7c1qkVreHptIx{hw9&K2sUJl`Rh|Qe#eVbVL zDCI)Fx#{<> zNDOGko64yD33JR$vO}mxGAptTA(os;OVmEw4%8_uL(eN9)8TpD9$_u9YcXgCcG{-B zgKGS=T@$Po$YwO_r#xBPM%14QXu~Sbj?8n58gq>AHjUqOzi+B0Uev-u`#nCG6G~hh zv;E1|tg(_CuwaN8X*!mEz{OGz7WX3xxLXo)K8X4uo1vf^SB~frlzKiKs2u-9gXmq0 z^2F*N5wXeIm$EqTW3qqMFq|g^`2=HQ?j$7^Zo|UZaB1Jt(BZBzoSi~v>}cs+aIfA{ zT2FfxPb!!$^o@yG7H?zm-~%_`%jq?yZsYAa$2r&A_nXS?^z=Nr6md+fKnODtw{J1q z{I;lgP2wc3Ie*BIES19$1{mg}O@=m4#o3TJo-g#tL_R3PWrj(44?N7pi*`6At#i49 z*ZS=QXYXr0ikw2jpDW7po*oX~#<_=Vj5*EX=1L0MoL@s;1fCCfke2k#LyyWPKBPD* z%4?A`ci&{vPOX{xt|p5&&G<~m>2~mOut$?a-jD9SS9ccH!ucdSIV8O~R#wG(FWS--^l@N62m0fTXA?&yl;G2P41sYhH!LE3L%PKY{1W)pJkNcmCm_$Z zVdim8mfuwSb9%gayV*TQXRMTpZ{hdUh00rQMC11TJGQB=Us*qX+O4?n5GB2SUZg9n z{$j;$ng&{^%KG#WHIVH7=}`bDi5OqH!?1Mhw;MC37my%zqg>WI&}zJ|x(} zJS9f9T)47Qzlin!=Jw%2FseoJyvzdxgI8o)|Ir>2MKzGB5ZTwtJFw~hbBisJe5r`K z=z&!Stm(&LMpxI;bk{F)nAKC&gihr|hpPqkuA1P5yI=9YySt10{nt=Wo3c2LPvUDT zv<`WoV@9#HCYE^$cZxHLxNN=P3kN(n5p;LJ!(?~4PeiyE!^R&O+mzo+5Lgx;wX@g-BgYdwV*jI~7O^^SgNUh9hpv zTg>s>Z(5>c-;AqD2lq46YrjPqFft5XWmj!O=&2$#+|- z*#Po{AiS*~{uwit|y$wPsD!g z$23K%efV0dWx5?2)tN%lFFWq^sQ1>=ueWmnP*wFqVVk_yWf(Z&7uQU%F|S<;%BqIH z-9I??qcOXvyyJAre+BYb_W|*aszNN^&l$zK;<#Hg60kPyVn0Wyva*$jQyx=BTF;y> zX}4+1PFKF4#WA<~s_@9X*O->*scLD~llnAV+j2aL472Gw<*VkyexVSVeN>vk1MZWK zq3oOG{zI3MgwJ{W=cm86-HPeDy4U5v%i}n&y^6^kvmx(O`SQHVtsF`rA+U<`!^vM!hv_;xRV|@VIQR_x>t7?XpBMpcU3tQ za20PUMg_e;z^hnIpZ4d8{Sja8byVWw)YIM=7Ze0vl4p^h{_y!eHx^G{l_b{||6bpp zg5|IY*)LOQWk=j5FBoOVOXgc>T$##0sO?K0p$`0Ci;&s;X1M_Bcm|~W-@R@#0p@U! z>wTW2AkZF+M(pt`;C4~F*8TS;fI^1UwQX--pMD?EZTehKyge>;`@Hu8kWpHnP2)t% zLB%IUVg&TUr@NUIi@U*V4>tR2Ob=#oZ?`0CmqXF*a+Pt8iWKx=8#&iEG z$Z;HC4%F8Gq6#@^S+je(UDC&q@w}Ol__S(4{@|^;-@K$pAG+zfq&vpY7|k=zbMn=? zrhOj(ySbwEJ4a=CHCvfXn3zy-$m0bBe}LIjSye+VfV{|Yu7Jxs zDuOZrgwY2_;2^}Z3VieX-KzB%bST{$fKV`_K<@u~!C$oJMrkijN-m}>*Xy{cr}kC*1%Q|l)VcS-WnQQCLx0I&Nvf9gvlLZE zaOX@1(0=mHK+-ZR!{oaiK|Oj6Y_X1JiiVH@t_9q-D&1<`|Lq0gW7G8GNo`(|zWatN zT|2-s5OJA>4pX)5xEEAaEafA=3=iSke=CaRfj8b711f0>a7zzh|D;?_HLTC}uW2aRHw>2t^pQ!Lj*}Gf-m*g@L6xgP0Sis#`aC?v{ZWlfuaAAM2{#p;) z&Romsq&UGH2(qpj;}H!x`2rH%0=cJ%1-i z5Pd*0nJN^`RSkz8I*rH}#5G1#{(L9A2`>`hGtC?UUB9Oo3ykiZ;6DIwb+9JaBWAZy zj1_u`B7Bie>IHD1yl|$-h(FM5A2}-W(xb`Jf-^Ckez`(t`~1pBrkF7bHcit_<;r5? zdwag#Gdqgk$(M?p!?$Y-z)a!qL;Dw4#SMy2;U#!UgtkeimsLA~#syg&!ksYG^*~lw z@Y-4&oiM%-)33YS;TP9zb0 zNTQW_Mq_KI^G5$5mLy2*?Sw=h5q1jR7Svq1(>Bj>1t7X!RxAxXy~=@C@xUk?p$jUy z?|z244fO0v;)TslNua~W89CZmzQ_w;3tlL+7DHRj1+Zq=oP%Ht`Sk6B?spXM)LA66 zEdc7uo!V@&pRiu;Lq*hWFaHS%1^47ab@73}w+uDM3V^L#) zrR8uO^laO3GA`6)zBa;7n(I!R2bpM-A84>>NbiA_%JChPRAYrBvnUNqa&A%z z(pYAeT}K28f*PHUKn9HGY=RkltF(a*RD81M<4GIoma<($nHmbDn4=>5RaFs*$%;`g zAfo__^MCyxAIrjwGE%a-z$Uy#yKDSAUW+Nix)cOWQi$%VFB)P#S#Y=v2uMouN;*kz zv}M`K!qp`FAI%yT(moM>C{28bC-78_Z=msxk{vlr4#gNTPSK_~+0lTho9Lxj*njgvg|ayJ%GC&}~a zmCX9R+p@V(K9g?=Bk5JloQ?*fp=|}b%YKvK)kKnQOf{?6o&fuMiha=$x^`euS_yN8 zkLR3)fE8Zmw`3p;bkPfhKuBS_GJYT{CC9iQ-?4$EGo|kY!YJ+VYo2($wHo}B!hKz> zrge2QDqXwEzV~^1Sy31#asbt*KkpUutf)lsIUJMqW)AgtDY#a3Tm;|Esc`ecSGX2C zV4mzotVhOFin$ZP#(y1nl*oY!I0om-DATB!?G2&faMuq) z+seDkGa|6lg*IIfpGd@*Dvdy5;FQAix^HQHAW-}&*i2_FYt?re;o(t=;R>dSKD>DM z-I}v|oQ0dl7>#stIM;-gyY6A!B1vVXn@hd6mliKljQL2Ycn+t$$|7E>uJPeu6koM$ zXFdR(>p($rAhW5$rI$ggo(jndJ|&+2;~sS)M$^?OeH`biMbpsXPcex1aFxbz@)o>k zS>p9w1zwacf$TV>&Nt&dDWj?c3s6eP)ucl*wcj2A)E{%x?QK-imIF5ssFmD~I_x~f zF|?a>TVCKMrgrikf;=83drrlTlEJ~c-cmTYdD%LrX_6C^lT59MjueHujN^hmpvEdT z_2}?(^X*)M$Qu&v^=@q%gB4pASh;Jo!@Ka<590(wR~cjYt|V=um-L*QSIke;h~;9{ z=LsNPg@Oz4s@s*8Jn2tS!(Zf`auIfC6Hx5)&xELvW$Z}H@uNX|xkUU_IaZ?ua4ecz zdTuLw8y#*N-|TmeuCAH7Y#5(FwT95AwT(jpulGCFZAfbRx0#=a%e{h&0#m>}er3L( zJ1!M@Zc6e6AS!GYAY^?_VJITMGQu6-h5G?WhZ#t5o)j5E-z9rq6GN|45EilLizwX!_G{ zS$hlD&D0XQRP)2qb=qUfO@`8-PorblN?m#dX7=#B+y-Pj2q>Mp^OG8$WxST&6Hu8g z!r5)xdgvR(GMKx@ME5pq-5_qbst=IgsjQjS>Ro1(kxe%d$sX!Rj1_Xf&Qcf#7#S8D)f5}3 zm|dnN0Zjs$M?}j+!|Qs71x?lf56y!L0b5@>L$O&uaQlJ5<*kvpj6z9z#-i5J6qq9B7ti)t+CSR`1LoyfHqE~5e$)unazTJ8_c;sKF1AC`|J?V3CgToV#=%dXtvPP3q1f9x* zs%Hv8RX}CjCLYeKR0SRU+7k zSuMJ!W?8ol+*fGsq|#)>H00on>X}?Q7lO4CCfa_JJ)D)59GWdnR!*T{Bi({A6rpyh zlHw$Q*9PQxWqBT|Y%*la=z}3-s1P5~BK1j2dYc7?^=HMCxVR2|2~GIbC5H zT9cDez?3Q^W^7yNZ!n}AQ|*rl_RV1boljb$GpDUE&J>VA^a5>2os=ct z==wbWtr8^(N9979u+I&qSR}JENa&6_HoK7I{riF6eKZrH^10rTKG;$IfP`q1rrlTw z3jot^p-=?Sp)1-ocEJdBuekg;Hc zhqbb_O!cS;^}*b9LfUC~s!>vA@@}OVZK)!h*`c3q2{=K9rE@2iH!#G*s{PCG%SX%4 zePq35|Dpk&=%Tu%nC6c=3H1OW4wCpAvkEG^F}9D1q|-ece*`YZYPo0TN))XNICU0EgdT(l8HPy&nI@xWkd<#=03CWCYHT7Y+Htm-kmJ0 zlVmR0H3<$Bh$3OLk@|y{>43M9K`%bpOvjTrJT}xH@@r!>YE6}EsD4lC5&>S|e(H|& z`zKSTa266?;!OR5wc`=8z&0l~ z8JGs8NTPvB1{IdpgQePfBg1?tzGV)nDVY0?d;&Yo1N6DRk0D}w@dltNspgz!v=NHW zN6;a86m^N@1a25fs#b!tXI-hZlxX464>aaRdx^PZ(xrjBMVM6qiE}lHowgd~G2#h1 zD-Ju~!W>bvNEWLpEXP8OZ2LCcK+kej({2OcNnmdX=-K$j{-si4aP$J@E5ea98Sgv^#YJ zlCG8rjkrTfa6jb@qelS@VjcNYZl~f3x{X0oIUD_UxoBT@`Pbnn_}WV3Kx)|eE3?Ee z1BB6CEZ^CLZAoI8k%AUPsS$ZpDq>t@UMlV+2Sgl5;ov@)&tPQeRcMH91oBv~!X9Xh z?}cvMVG^(8pGgJry}!~_z&jh;CPwseCdN0dJB?vo$|dU0gz<{OEVx`}n~6Jx-R7qf+7&loj7ab1(W6E7RMO0>wJ0ER<6%sj zLN5-qVY!+*UPKq#Ha|hbEp=5niHaZqcH#NXWI{{k$L0*8!qSms@+=uraLa5Y>;YZu zNA`}N$6twK8CGB@3@&BS(OaEQ!1UR|a-&Hud-yXRGzU5^;lBjZ#uaFXNkwFf5|i^p zKD01243kl3JBV=33V%i?qf<5i94F9d|Fa8O*o;NVH=EVvP^I&fEAm%e5x-I%q?}kj zPiWDVu*dO;OE`1{2;;;$B+j(*-48VIXkof(caVUJQ&tq%s$e`EOy5GYo+3gb(5FgE zL7pd+e+63i{Nsy)R@LqK!t^?ZPcr2*@OvAsndK4}WSbmcbGe2iR?6tOPk-MBCxsyT;I^2+(0YFU9Zm%YNqH1A z%b~A84#z=_MhBzGL#u=+?Mi~S>ILrMl%Cr+z526sW_^ctMnZ*=d#XJQ-<%x6BH=IIrTg;Ewm{pgoFHe7 zzWsX%WP7VAHiZoQY@t`nN13Q^YXpE_ku!`79t6z;4-qI6Q7Nbe|Aq)55{h0(+$}_C zD~te_o-de(dRXprY=Tl}79R5XoNGAEzy!qxp4T$)$q5B|6%#`H%lOTPvD4s(Kdj%z zG1K9z_mO%zLnvA86`+p|zGH!xbD^k4;i2iGzjSuO^7*`Ro9A2|QRlIbuclC*>scNW z3j_$6qB$f9nX=g&tOH#6!vzf^YB>cYVCoMGaClz3n?+CIw&aUpCxt?eFEt{z%j|jM6nPRhB!-k^V}K%~Fl}`Ln#@7^T_< z)o{X9QFm*3HZJfW8!7y_@cLgnIbu@cYl&{o6{E^UBAvQ34` zAExVW4f7)idKLBW5P<&FfDCMc0|3F~E0j!JAyfdK2#IPD$GoP}!apJb82~T|u!g5B z`Otr5;0xh(p*-}Jc>l=;o&a#CkF(sx{jV(gS4L1Y7gPJkHwiVG!WTWzS0Ea|H??Dz!ODtu+)(LCuNlL0Z}_;h}{YCzcMNCMC>Ef~fl{tceCnCA3t8)DP6bO`8K$X;q=7w4RD~AG8^QgcR|9{g6l?VaEe)n5|!Sew~;hdU*JTFf0?V0V3fd~45mR;<^ z{o>aT;6I#MqOn12eFKZ`RqyTrgsBae_wwH>i z*a^PesIW-6U4LE#m~NtPB=Q%UKY;sq%#}XWsjRRd?ODtkS7q-nc`l7qw8f$c(vym~ zZ2a3DdzTT@K;v6L2mBQ&a$FaW7OIv2qZ|K{jx!-Zz$29O0M{g-M|l702^e4xHw5IH zE`SAi<^mD}@bjjM0)O+(tc(oTnoXl^(~LNP4Bc2>%kum>06*{~b06YgLnjE1$b+Qg zqxi60bOUg!4S?DF^#ZT=|K8~Q1K8}SdM7^p{05lK09H&u8%9-IFAE_b+b$%4-ihoZh??ht+NfV<~AT!Tem5io&gPWNEc>Ag_Lcc>)Au* zl3|+K{=4hRi6*QN&@ z!m8Z=>umnoI7t^4C-6v9h9rT2CK;7r_391m4y@PaRTR!#EBC#`Hy{Hd$*13Hy#MeX zUTE8Qvh6+pQvwF@tQi^&ua4YTQl5xi`VpAAbS?|WOzwdSqzC?Wgx?CIOIiS$%1@xN zC>{VyZTl4W0pRxk>mUV`5kEi)@-#{S+U2i3JLMQ?8y`RsX`6xNk&*&%IR8!KtTcTA zDq79=JFO2j&5K%x1?lEaSr~O2fJW5{yx;~P9c}CTTn=E!GXDSslrLhBwB@y)&aO|!Dxj7LFFw9P%U=cN37t}5%01I(08G+lfp^7Za03bkAS^{n% zM7^bDOzF@*!?BO8>wrJ|W0U{eD?o|a2xjNOp=$s>2eyD6z#pmdaTK0Ik`&ML_5c6A6*7jPfT(8uj>=o zLO~84Zvpnw##{}90GLq34~dEH!}{@erz>;A-+*DyTMSvvbkm&~xoAG`#tjIdge?iq zDddVdU=N^%vOGt{hs;G`HRBt}u0G?p!|&;L6@7$!qy)gA8e$Q3URP?Ro+VNrA1Cd2Eocq4(ei^NSDc7O08;fGN#U z#$ohnqrwyK@jsOXbr1sxhkXI>vlNx~RouOC9#uz&hy-o89gVLJ;O=Fb8 zTb^}6Kwilu zT2K&X{j2u%%N7JH6;#{+6L}8Z@UK-|_|*usFy{3cQ*xQ|%RYzzTDl58I+ySL+?s7G zAt~3;2Ow-n2K?G%+YToWM0;8ll0#!peW%sWMvXXBN6tK3ClQnWZ$S|1(NQ=S@kn-} z|5^umMj&UL8zrSmM25+U{%|=&OigPv3Qbrl>@9|?6w;o{FgzFfp_Jks7P&)h{U9Et zzymd&j92?M5&U%onrcl2upg(N^D@E>CvIy3uG_e{wWmQNP1Py4pg5bI>NOxdN zG7hF`OQF&F2b;r926-*SjHeN;CEZWC=_YE{N3{SFHK!Z>%8WrjU^C@JOR=$3{M~*) zRH@qOQuVa~{wNhQUAjy?C1Nla(Ae<;^W4@QC59=aR0%?`DK~V?6J06|Suq9{aWDOw zpJJ&?lT)l@8UV>V2}CM;+Jc<{T%`H4I6vEvD-^>Inz4Ns-P>;@ie21Ul7^*V?iZ!Pljlw3z6Bh9Ie$X`&)H8eL5r9$ zR8MUL2ps4^f7R8~IaWA$VPghW+sc$(A)DsUVQUV|orb~=Se9QsSzJ53*89Z67!Q+bb>|2eDmX{l16em2yI7 zp)tM$4udGayP6U}R3`HH1>v-HTvdDiP2!H}w?UGn|M|2iO7ldjF`)8S$XZ@88sFJx z1Z7KH1N7K@e^*xH&x$!Q94YE@+fJ;^G1>E09cDmkVsn|IU_^0x^gdfzNgmjSHkrRn z6ipM2t{`+7XB=$t8O6G=$>!JyLurQlQ!KEc#I!sBeuU9nXzK?q)R8c?*zHrJN|VT* zSk;$n0aJOb*sZ@?)Bk1fkOBVjHraS4>%-|uT;V0*UIF)ZHnaxWhHC*5b zq{>~H2Zds3p8R|PTxkksG^@F*2vcNhIlc39A=G6t(zjD7qcNLgKGT};;=+C!m-LZe zAui#PxVZ+Jw>TMJWD+`o)lJ2Z=i!Te-9E7H%fHYh%S{6z!#*qCV^mgLi~ilvD=1LJ z5qAZRE11(ec$W{5H#Y0FR(HuyXLAzm7KGVvgX2^pD$^Fq5~taWPk{XqPVUz*#oB6V z$SN`hS|L&e-UO@i3_22L0g)X`7D!zuxF)Ws7I_QnOwG*Z^k%ib&*jxJanDcz$lza- zoxi;;dWsOn*USg7Qk9VCcq`c&3vAmd%yhjT)a<`K-0@gfFLo|5oarcFuxCuYs>c!W z%bpDEz*kU}+M5{cFh_x(39Bx!%`sF2N=Zae97mI~zGM>+gQ40`)z}9DXDeX`bF3;W zoV2J{&d#=GPIuk~^&Jh9FZC;gL`rprD=B`C9Tz?}f+SB~LFQG!>Ivb7wBw|$z<-d{ ziqzK{#k5+AH^OkPVm4OVZ)PwfY$yhosL0J&=s%QyAyQ?OV>m3&%;?)C17&`-9Z&*y zu#mtxD#q|2RbX=eQvI{Ee9s_mVg8A1Np7&Xi&3~C$9nc^JGz+8e_O~m#a&Ug_6)Ur z66&^ZRu4OoA_al5%|L_9J&~(?a4Ufpj_LerZ|(-Lv{914iXcYAq?5DUcwr zgL|4J-$t?%)3>dfLeY4tQ9S2AvOycUf1^CX4a3v)Rt~_9+j7_P;i=ma4kiToVG@Go zNg-=23@3!Sr3e_fVU)tE33I7t8ai~cD3Nbzbr}m*h)6~8*8=K~*!YCh1?>KBf-@QY zh0r(4ZL$OF_xqFaFPLlcV7JvDIVB84_fIr%RfXT2HKRYA-+_UA1!<;b=WMzP#;L*D zqA!u`L@3`1HT}M}%&Jnw70Xd1$TOiKY(n@irx8P| zTRKS&pNq$uXqIL}AhYRg_bDwVzl8^a z={{>7^|5fPIlYtEKV8{fKO7i3u6U?CqhqLsiXuyV(Vmo^$f1H|LTxY?4ZAIUq!FW% z*XOpNT*I7j?W@2{VHnYYe((mXOD(zYWjD*D+?VNnL9BA&+6AqUmJ&!FQ=(<(s%N+u zjVpbeVTVxXHguvuRem)UPdK;6+)Z5HHYZWc0pV_$QLkl8a@`%x4Y5cp!4xyus+osq zt5qEfRx4LH2Y1;Vv4RG*CObOt?g94^%jMoaugJ+ctUfKq@kJM-`AW8k+zN2aBEvhy zhb}rIZB7Y!qOEnwaVa~yZx0W7a9gmzE?=^BKI`X1`+&uWZLhBV_gfuh%1m^AdCI>z zTicIn)tX9Wg@Y(vTw#ucq#Z$hDiyd(wOLo(mUZWH1cCK%(v(@WloEmg8)R1zZeQr0 zna_^v`Qdj?8yH3?-PSeg45GHob!^(fKskczP= zODr?ClXy-WGmm@zb=my+9S74q96>3={Vee{fqM3;o;^>pxfEj}tak%cLZX};mHB+L zWZG}$pvmIX`pzYwz=}qz;513-$>CyMA}fkYK5DS6dj|*EHA13wsh8MIr(7=x9C$q= zha!4*9R8FfKO26PZ!`cF&q0BxnoUF!aj)_E2Dba+tR!Fb+qHUI)pmTnPGBGHmsZe8 zdT2r4p{R}!)WEQr=+7JJF;8Vsd$2g8MBO0EL#Z`M+r(??!vN?!h-9eEMhYG4+EckW zCCQmYCaF`iK`COIK|zmY!`u;!Zj@SY3OVsmUfj;1Hv1g)Chg>0kg%fegO1NKjF~-G z`7zbiC;|m>hRu^QtV4K42b%cvyLTvqe}5MMq3Uo@90JqfrH=%An3Lp*^x&?jR}mu> z9Ts^${mZuf75EsnFM&HYg?#WzH}F)&MZ;P0Tx;MoBDKdIUw#5$e~-N(?v z4{1VF#NQ*v8RR0->KaO-vqn6Q(dcz*9a#cfI!$C|_%<|4q_JfOK zYL|2;$5y7RU!0UmX&DYtPf{&i;VK>>%M{3!&OZ%U0%_Nvr5P}!A3Xs-sA-~;ygwSs z3Zyp{f0SJ#=OaU8^e`+(qM#G|HgbUYQDMU7FtKVacXFQ{r(@W&6_cFPpGB2(%8=02 zM|Q2zpW-+e>g%EnQcLKW(T)GE>{e7vFqtal!X&pjoLyfRnMqz%ZI@Q{KH#Z5?Gs*Z=oMPFAleU>l!>P(gD*)} zbBU~_I@%mCavB54?n}!`S;!TrIxW3FD@U);AhrJ?>~j6_yAd*GLBB;a2Zak<2=fa{ zvvXt+9>jp{8KQz<6ll4Zf--4_blTxltA=#c1|8p-3!5zK2-ujj1%GwhhTM?kmur4$ zArf6cZ?Uj~Xy5!Htf$nj*29XXYLTjc+bD%3vhyz-LaoXjx)-o*xQ&DOs*((USc$A> zU^En4{t$!CYz5b##+GGfOI3EV_W)NU4cm*9USVLGc0@qLZiMADt`pgEiQEqzDUs+= zjMutffWcK(yGk)J>EvSKRMTE+lTyd4T&&7YSiElxNzHn zde5ZZ^_`MG;*X?kE6;HG*^~%cJVZ+`(~v5>)o&IIUM*~y*YoTM2y&9;S4%$ZMK~bm zaMYPZP=&JECJV!ai<*SIov>(H)ilt;-z^P=m-kAeT8J8aXvKetyEb(ZIH7i%R21w> zd13r$=%W!CA60{J0i8h`S5hy!r!?MIfKm$BdJ@EywEQXF;9qkb3L`$PnnWQjYCCsW zmem-vB&d#D@)2!WR-y&BHJF*a_@P_Gv|Mjf(`d!^e_x7f3G-sqm3dV3#msdXx8T}Z<+6S>* zbZA*(O2Qncs$uz8%$9 z^>b~iqkhjOq)lQPy!ybr*6oQ(0r!L^JfPV?A@V2{%KRrGa@Hqjr!yrgy!e@4?Pye5C&i$Xs{*U8fG9`?0qs?aKwB)Q9V;h@8 z#ME8nkP0(Ga*7V9In6Q2J??08K30wuBPw@}V{&qNl7SbajWTnL?mX4E9^Z zBA(R^gecxIqOF#wB3cQm zB0cI3Nz_7f8)AI2s~m$p+9P|4SK1K>!xyzVt4w6*SZvb zK9NhLXI<(Qu73V>s&J386s5cvhmqCQRJgEV><_0+%8eA&$@ZlkjXJ7tCS>i#1ZnIk z2Zp1Ud!%lh=cC^bdEW0Yk#45ruld0r3rkXbtTd9F+;@NMVVu|#IlT`bYDC&2d;h3= z#!bM9I)4!_R#BBgB#Y)fmdxeF2nF}u2yE6GPLOHXo@sgLyv-?f0UE(jrX3Nm`Sshc~KVLSB%+54a~PU0%jqL;Jgb8M6T)_M`cp%jH?>&>5Hsys*H)BTXI zwQI9yWzB_R{m3zNPZ`&Lq0A;$T^Tt^+!E#^zEAkw%S6awFVzOG8EQqC$K;8CmlnKb zG--}SIhc>IU!+~q7I#~5FA&JDpX*h*(X1xRl(lcjddDp&R*va7JmzkfZ@N1hwb@JV z6l;)l$}86YdX4CqS0|k&X`Loy*WS2c?Q=4>P`-i{{S6HrX4^d)=h$6R5m6RxR733~ z-Q;d#*_k&Gxo<($$3%CKNKD1zc1bEy<{4tt+I~nxAaO>?5QBo(Jq?%L4=w%D&#eC@ zME#6DY=IA^ot(f>6|{pxaUuRgiU) zC9a{{+vNBXObH^bfpV9WmuOOq(l|&qQPX%90>tBW7&p#RaD|CUskYtbInvi2>@_^y zx8V zFW6ANJf$-zaisH!9G-sGF1+N0n}xorR(oWJzRU$EVe_A;%b#OLPfP19`#J{@8cXcH zuc^+_=8Fb30}TpPBxd?KdGYRt2Scm$4pzsisLip6=+#!BL|%3P-aBHb9%p-Y zQNy&=dU;NGPX@iQr~4M02`npt{bW=Wp@C8rmAlv#rx|z_UjKrP6H|h^%S>d>pharZ zFVJ2%zPjzU3#5G3X%k)d#>lg@vc{y$-L`ZW7v@9QwkuhJ_RcyWuf)ST+tOG>Ttw## z`W2M$!$l}%`14;FZ^Tv6?zI#uIkVB_?9Orn1_P&lNs;t<7n!0rtC4NywrH}{y2KVp z*U|~PDRWe=f)iIgse~{&J+F<}-@{pcGqF(BaBRuL;X6~8P;plgz7HV15m-BdKoK7b+|CSoOx`TOM-6hIws<6OJK zDHLo$8~53=Py)zTSiVTdw5vJdx@J{zJeWQ$BPgcpjyGPq^C|3g4mIVRCa-lhT&luK<;9MDn$)UaeRI%^m zEw>(2DIcS`f`0gaootM&4_fRM->n$t`x0YG`%N>i;N2nYnc$S|aGi+wDxWAMW(&Oz zu)#w~enUwH3C%A1uw8SN;PuX<2`6{Q{oMacf`_&V8amJeqe8DdF`K|?!w>!Z1aC@qq*uxxDTdg25QbFPc3y8=kx zACh7SW?r2U!Dh8M-Sk?llAxgtJ+KA4I&`SUv(@ANXvjNaRe=ML_*Nhx-OWvwOf7EH zQ`G?AIu4G_MH6>=QJXGKvnh`BCWBfY>*E&71%qrQ%WP1g_mldU*B++zTdggoYm=a^ zubZRNE+zQYC)(WehYIWuG+!U8D~v$2kQWfc;vhdkpvi~yDsA(7uX>K2Kf~63w^mR5 zMAe*;-m$)dH{|W2bm3VP&}8J&qu`g{k=sKGr|3XMKm+7AiQEO>*q)!BqTKPBhe!aw zi1*NlK><#R5X5s`=B`C3@&_eUjlg=7q_Zl$!xTteU1aoADd)eump>Ao=e0)G{}oQd z35F5rqTtZCl?y;L#?`I$`Nma<@lQZKU?W*JjeA%UkV_(ph#Y7K&o+m=xLbnhrPjh};`37&@^UPGud-u81@G8mTG@G| zBO|6C`C$3k#LbJ#aWOZ6RAw_gBJBF&^^xZpT0`G~e3KPSH@mKw)v<*4yazCDnASbi zS>(Viu#wrc(ZE#u6@b_67FK=$95XZxFf{;XI%Luec#d?Vle}9kQgOp@0PTj26!TbE zq9@nbXTi8*){Jm7d7j9K7c! zgn6z&^!wM%wdfzqgUS7cZSquRD{%J9<`Uu-f1V2q)-(_7_=ut*2U@`cS(pLcv3n-v z7h}^0=lMzsc4^AjW6J{IYXvD}|9;_sf?WG}9gEzl{Q8e$P&}^Eg{mfv+TF lKT#uY&5i%R$Nz?y>v&zd=-c%JIP!0s4e6leEu!b;{{YZoC%z4i_&wlpaPuP1^1p-`3Toe=(0wqP+55V6w6ckiOY)s(OMl|6x z3JN`nlC0E652FKLyvdBNKNrpR*u>RR--ru#;^cDWBwR!i{5~qhmVe86nWHE5R@;Yg za8>5@A9G8h-k&o5!HI2ham(57iVNQ+5Hsqv;!3@g(p>&1yR>aQo&HyJn*T`L`RLX! zV|&-+Oc@W0rqd zBo6HXGZ;Z8Dm;YbAX`FmtkeK>Oe70AhV^I1_5iqZyYLtWH4nXD2fYqS1UjZ{8hTG4 z%Sr%h1rBCMstM3CN6kYBM#aKMsPJG$}sBxeJTW~P;G5(nZ}EhQJrsEzeZukU zO>X~PLXnM_1wn`Fw)+EnyKv6h9gVF7Z6im0y9U^e`$bLn=HM8%&6J-*s6wZt>4({jOFG-oW_4oy)1Hj@^pnpe~Ky0XoA}=s5?7 z5jRwI)WC5CZoaK$Z?@AaUu+j>vr zAPv*PT!4v$kb=ABwmiHf-l#gL8Iw#&b3A@m=4(<4q7HHyEMJ-?Q-E^Rc{s(uPWe=H`C8S< zeDEE@JG2!3(hegqmmUhidP zeDnLbmo5|W)idlm!cYgC1*~Xk*ElMUYmS}Lg3wTWw8YJ;YcLdvT{kXI^|EbyBz>Q& zVI!_OLLR(HL$t<)Txt#$%qdm0tz&|&MbU@~Mq~9;$d?fhpj1Q%<9nTM4ZoXnd$`_a z#R~}FbMft<=}7r|*mN{ifiS>y$@s2Ecd z*4!#c-o%QYPihb#Fp+oOzzRCcxj+3LaYOz?uBI=$$!^;ylxPx$eJ8rbL(HY;L_CY> zsbn$V%*ocpM8HB+*jYmMmaNNL;!fk4Bjr=~zgnuufNQpEVuWo(p^}FPOj#NZ0)(l= zEX8<3YJ$MiOFBT@ze=65Ivbv-iJ1CmE!*x=KQy5xd5|@C8E+3S3(^$UR?P0eoM{!N z{%w+I`K~m}gb9sA1up`!Bi*)S@${PC<0IHP5q~3Q0{C-K#{=NgWU(TkIU>`%O89ur z!Yq1t^dBvK&VI6{vCCzWx82*{omH(m(ARhO%bK|IQqH$Pbfz1eQ&hjuh~9y8+=ntH zC98aPhUrVB$1s(?NBeiZeKsZUhc-NI>@Oa#2K|-Zi;N-aM8%?B*&0fT|1b3Aa6fD2 z_dCqCj>{6kF1TJvRPte2(_!jA72?yx+w04=$3@?N9au_&Iy@@rZv?Iv=jZp0ybo%c z%j(akjU{WUKffBisK`FX59`WeBYn@?xnEkBq3^!v8qcBUC|t+HJ5*g+YN=>AqQ%9V zYrpkwM;uDAWkt6u_Iq)AhH(61szg8`C4OH? zy^gbe6F^?%Hjv47ouq=*(JVsAyMIV{olU0|GuLxy8Q%a3kzk%KPH_=A{kp?wPRvYJ z4)vqN8iMJ!UyqwCS66jICf)5!?*~zZjaUnltKoYL8b=y5+FXLTuZM^VKRytH=_gAs zvvoXl2Oin6{Z|t#s$%_Q==`m*qLi zjHW^k)-=4+&AFK(U2TbuC1@_CoVU`v=G zOH6#AiWL*oK0PsM7P~pa8KWKVOJK|H8FIKp!yzT41q29Jh^qRy+&iZ<+ghbX^DO)c z(<3_vtU&Em#RqRBjoS>v%TO96;_5I5p}@0Jd*4A{;6hRLlNl6sTtkBC+N3rbxmwPC zpKS^+F&X(?Z(35=QPo{{;&Cf4s~9;C3mgKsh{{0wX3tOx|HO%Mum7mWsQ4xG^*>A! zb6wpgyvz7yCR*1?^~19XHMOtbW{EO=PWm@|bC|aX+7y}}ZuS=o)pO%)fVCjlV2T!t zOWfwWx_z@>67RE~?q87Z=2WOgoMbAIYX~=HBAzJHz9%43+5c1THA&*EpIszr?nwN) z+?Q&`SWm;QU@A2fwcIr!oA)LPIBpi@e24!Dn9VDD;v+yP~ zhn}>P-Yz-NTTzkmwQrrkK960U3G7*Mk?`N+o~wAnIycP%?ZJ#`z>PCM?= zsRU2!Kq}-;;a6Yo7r8z1$t*}&Uc2(ksJ=U~lRl=!_CR!!-wx=EhvGitU@+~g(~Eg~*37y&D|R8LjlK<` zJ=w*$Wb?ifLMUew!#I*^m8K#Bj@$LvOY6;HYoT8B7QH(JMO|iQbp19H=bDjCuTD!D z-!KKfYuSPpY&6JEQ!4O0sYnRaU{_jdiO|+_|5`(t8oJZ_c`O)S`z6TqU}Mt<6Pd8U zL!ZbG7NSp-%Fuq=Bh@q&gb-ZKYyrm!U~dg`Fx7>Ey=^u>gb2V4U_9x3EOvVKuGyD} zirmjrJNlm3_3QXj`j{@{t9XiL0K-V9Y zTnbA`JctK#_jdWw&uWJTPnrZ57iqhHc&KWqvvy=)UNtP}eAW8;LmOI! zGM3qR)&L4GhK?;L_5F2nfxVqYsfB47`_^s~e!0vYzM(M^4qwnhMzqR29rL1^$9u!2 zfcnqtzqTfOxM~iFIDF$~Q6l+2Wf;^8+uREH0`S5NfCw9j zJ?!bDKk?qvub}RAiD&g_mb8TR-Z5>3mbzs1DgUVcR62Br8U`sLtR~jO#z&au9(e`j zlm_+%YW-@G5)J(ioiQne5B7FXFOZdeX}7~Qa?VpsK9`H%)sl&<7npoVHCqHY zzc@0wa91s5(A9RwF2DT1A|)4dVjQ!zC_$CfK5gqpn*Sdr(u&KHwGdUEzv=8)5FZgc zLw3;2w3n~$S~kXIB??VWt7Vcm*DI~fpHnIkU|hMW4%v!yVo=}m0KFz@YQpz)EIwEI zP;w6&+N+l+odSbHY_kPQw<#)e181rV*StvU2SUY1T5jK-H`6p{@9TLNItK~4*!{ya z%vLIGY2=~TavlrV87_=MgO29Bg*|X+d+vJzKO)-_2GkfjFVZjxf4UINcDfSGce~m) zIVfas{3<1AB~49^H&yBd{X;TUTX;^ZAorvCW?C)Jsy0{?jfZ9Nb&3F%ZRVf z=?-*^V>IZbr?@cNb0{53%Xw`F`RMeT?7so8m{M9u$=y*GNKU(-D|uG)JB|ArE{znX zYXZco5)|?gtuQ5E#)Y1l*3JKL<87k(3tfDKEv78yGul5u8mEO*usJ(GE@LfcGNL_t zg{XuI@?+;6@$#4-$BOs9pzM2*V*MvIv0{rYSHL#>3elYEMhViW$kQDO_JpfvP~=gf zLmRL*J-f#K?T4$PL4Tg>H`Z7F)n+a8>Q-Lb8FUXgjw$9pqP%~NkKn|RJ-D(ysR&DB zkvxHqBAT5;BU)<)C?(o29}YtkG+pJgH)Auo8|G$C&pu}V5Mkv^fu{eX}^u#Y1bZdr#@C^ zV~MGND<}&twUE(AMeN#P$GY^$ZhMg?qOAp#FX11Loj;8b|*+WmqXN21>!Q|YXA7>?Q~yPoCG$=6oo)2Sn==Nm_2oq) zLRJQV*!kG@ngsD(ubNLsoG%xg)eb#E26|;RGSI#MR)orAG5@F~QK=cp1#j0#6f0^U zi_kSj5CEYtKu1%X=0ET-itIL$DXQS9Jk_WK#*gIBip~wR=!_Ge^fiE#TlH42 zb!b0a9#!4jVy31;aKyIRckBE9l3j!|x|Vy(=)Li_m*neeCO2TV%>GAJlFyU8?OThR zZ#?W;eChyz4AVB`18+nW68t8@{g!h;f%UedyN%hS%I~i6wN zr0p~0p}HBn;XHNBS${xbN?^^VB?3Mm${1SRdO9MyxLc5!QA^$w!%R3o1PQ*-SR?Ip z?M$<8O-T&&#!{mGoZ>Dk`9q*8JX7d6s&BQ^_rht%*3$W?e6;-IQtE^jWCQQPb=q*c z1^Bu+>{G`6_uzUwq%t)+keqmm+RlMPn-vSKTh6E6`7sHPU(A<_ai6Y<+RXd{M+jEW zo(PUVCmUJsF#R*mmEF%UaI8<_Q+VJed+>(nq;Xh&xi!dCr9_(Zo)4EXyX`Mo??-S{ zhMp2wrMe(=(ZDql)yS$QpP{uE|CfwBMv1_Aw`pV&)>Nw1b`a_~iAvfF#4cl&7#$SN z+QJ%LavHYbdbSG6^GznIu-TyP z?XP*Ap_(o1R{sfD>R+~GtqCsFT4{sntt?zBd{oep(BY}=sP5Mrdey)*g-co_ZRF@f zY}E3&Vb%FCq%3CJBdxk5$zGC%ne6*-01^7E_bqxwmaabg0YZ1nPOJkTA-768Vh{Oi zLH~K-Cz=^i1z2Rld?uX|q`xDcGj+z{LZL4oM4@L*Ng+=K4yW{Trvv{vUvhS83#xd9 zEdYr)z@j`JZX4U?%WfBvP>G!SHVBWzo*U5&ar;FHJ6p7YAnoe^)^?>ApP7GFN0|`c z(wi25U;FAC{p7e#+HY}g;La*p)Y?;qlX{s^`Wr>R(xSxFd8 zF!^DqB%gC@Jz=1IY1M!9A0pJ?pb^tPCbVP28`dvrc$7Z4uJi-)*KjLQMc>j)e=FtV z(3NPpX)V*1W8sarOhBLgI9J=96YJ3hh|b_ofrd-Pc41s>1trKDm3deSfY!v4S#cF3}*W&AZC)@z-+m4esn0y#}hKF`vNE+{#% zOxjDfY9{Ib=3q&NxQ;lOyK5|{M`W7>>z|+DE#R=0{C80QVb_AF+_n@=R`4dI<9wwf zD85V*$Se;3XdA2eNdi~);ezw1if-pIfu|UE<4z9ql8Mn;y8yrWki)wqE|sq<$qc*{ zHKJ=G+u!4>pNF9@Ux_l-T>~6~@)|4S{aJye%9?_C-g}PG=7;W9Aq>Rae)#*YJx||k zSX839mhYU!`_gKr6aH6+&ZT}TX6xAvtz&4svaD{t>OD${Zho7a{Pblj?>QkZ;y2DY zUJBICk^7X@eoI#`m)YMfxwp8}Cf!Xv|o<}B=o}n)yd^1Mq;IWx7P7S#Wk!c(wisz$B*K&bsMon7fIhkFzbZ35tW1^uJc1XW=9M zypw-tWLzx@is#pZHGGNaIjo)yOomOFLu}OPLz?Taa}e@Pl;P2R=U?`I&!t)&5#<(V zL$z(J=sPzb7_`rEWHm4mVuF87bl9W#R7!(Z->*>R&sNAU^Rg$^qBFX6Q_oKvKHh}w z>+P@4UF$6S3???h>bTwAGZXeGf_nmVY<4JfN_J;x9G@M98~h`qD(l%SbJr0>hgMWr zVS_||VU?JIQB8%j@Jn7ZV-ryMc3EH2xG%(B!~|vhV%-7uIk;j?3eDE-px1W&#CdDT zkw;p74xSqs_Zag}$R{2NOa4?2|`%vPg?x+Nve9aCp-HWIU*zP}Ky_zW=`6{%^HQqb^#3#`WV~kMnFL%Jz)k^R!s%2RwW`!a6@6Md2`6!kOg&l^|_C4Hcc?&_3Ovm`L{>w z+CPoW|0`-T@@xVSah`SY=;g7o6$Ib8{$(I`m+sP(%%YjvVwzoJk-lxs=5tu~W~aYm znoXrrLcu=}C|dMT_=%?Z?fil^XZGD0fzU$C)R*w366F1c*f734Cc_Q#Af!$}sadqc z`36WP(tPFs3T*L-^X{kOdLHEEIp5nHG<`^-CdPYDk`yRBH$R`|9JsY#-j;Fa46vlik#Brb#X>kfvb102L&wr=bf?_^#5%-Yeoja7lhwsH z@U)4e@|C`t+H%eBPlKt+b&X<6mTgoZeqqh$^f_~{i7+jCj5FKzIuK2rEf1pNBvve6 zQ60H^X|cd!?5O}j_KBCo{+ib3U&jwakBQn2%vnF3`rM|YR`1tZkOl0ppb~=30Pqk7 z{Faoj=PCfy6(kb5v_xWE@f?RsK0Jg3KcUg{D}x!)*wA1$H9 z&tMdKrE3i#KbT3J{XzwUm)SaV0(>reJsF>4IBy*XDE>_nSJ}s{5r-D%0wGm{Ze0f? zd|F_w{ior3=UPGfDYW7>7y+Uf7Xj7^tjgc##;H6j)(p}6$arT%-4H+ z#ty1k`;iall(*u-W~zPx*?r?c8|jMsc_Z&9ylDXk9thbp+N(|<;NmP(o?E-=4DfK+`)_vH zHAM(tHu+?=&Ff{p8RV`4(0V6`r>u^+=Ft#7N_au@Df|}ZBiw?R+kD?Cep(;@#kWpT zp!HZ?Ka@VQN}8fj0>dA_jc)TKs(h zg@KyKVg#s+ZA9KX*`W)72ADX{XQTqqUPuBWk$+r&f%`&ve+`9M8O5Hcn;*>KDQ%SB z8)Zm4z>!D~t?aPxqYvB<5VFBoj{Oh!8``l#vw1JtGQ5l!>bnAA&th9gML%SUHh6&g zkLSo~VxDKOaA2baWpOC_IgsLv1=%lb^N$EK5l9E+c2-+X#GQavzxb8hV;dl+RK6!| zQ~rJTa#i?rEu5R>al_9KIHQ^h1hk{I^F#uSpGQ3+6Yw%1Uuc~|Fh4ja#9+xkl!?@J zBUfYAeheijr4$E0{7&T?1vnF9WB9ev*3IY0*6UPxs`v^{Sq*gPOwvzO$2`ml07aq$ zDvhqJiMh--B1csJUb!OXWfN^A-Q>&jRr_fJXB9n6`yOUTLu`qY|1spCc*oyAlq@Ce z^h@zD>nPUB$*X13u$df`7E}p9<7zmcgWe<)KK6_ZKF>>;rc5=y9?<@M89c*wB~EXg z2jTd3nQaq5gHAlrd&qd1=fUc+nV`J|1+BAyXU7(}`_}tyd(MN%xf~=Nn@v@?-K>>z z6{1zEmlavlT|WGY4c-meaJRs{|#4t05J%J7JiO%*#p{&^D;fn7Z(JJPGC@hEMKj%%AH za}5o_18fM{bh&iIBx#%xg4|m+vhUgL?@u^A<$=f9;nJg7R_PqTDDHrbU{o>>5Y5O_ z(JMffHAcqdAaQypNV4}~1FNHEa)L*b?bobIJsgMT$dP&rsKAy1Owec z$4zmKCTwseJzb}#-D~?vGyK}lqbsJrzHGxhiH zHH{e@m>1XlC)m0xi15v;csb?JYYgs`0ZA+Yg6nQ);UEwv2{0}z=Ll3_>0cY%WrD^wl{tw2_- zp)PO0vhw_IRNy`|^#M?tR!(-pk)MKDZH|>`=80sCM(_ECs4y92xo}20VZe86d0mmY zYXG=2EJa1IBh}(>do~MH)AvA3L^zCKIPB#XD^_qvT6enpl1LRkx3UjL2i1oNHsT1J zUkvme0(~3(@e_^p=~qqlgi?OI(*J@EzI~I(?WSZ*4S8jyvOtb7ARBSX{$<0EP+KWm!aHb78lVP2%LlfZQ#udT|Mk61E;pg$bgWuL`6*WfY*U ziQWJmOug;_Fb3kUE?yZ+mZ$7zQe1|VmtUIQcwf!rQdTFG%$ljc+F0qoTb-o;2Y%zE6EI%7ii)iGH0y)L>U#wk=h4H`$F?lUId7^AsnHbuPQV&p2rYf<3x2 z*6V%8zP`<(Xj_cI-Ov|*!J0*KLfVUJS})$JP!q!{i7Xy+Do0%kX~tSR;*)rMxNM^l z8)2v^A*9pP>}*h&d5)1Y zl}lOoa?4epKZ9m-dNP2V^c`3<2i38GdxnFq8WV}eSpIZs<{Hf0y@mwPp{VU&Dy?KO zzWsrtnQgtttRJqMvVTA~I$h%EK2S8qt=PC@}XAw%j#w!Waf z%r^l*Dq6)hd`S)(RGq(b0(zHUu4O92kIg@~2DpC-?AWe=9n@H0w{d)WA!-a;c`YSo z(sd$LLTBRByq)1~9)(f-qrB-ClaGIEG%C+GpOonnrs2kO6{sJT4URIB^&M?zPws$7&Lg< z{&xFs)u?LU%2)HHW={a}uXlg}dA45CK>^45<&P&&T$7K=T`%c#-@z!{gQjhvtHh7x z#G09uy_#c5Ar$nKH(CH4IT+3@vdJ2I8{s8PV`hgDxTz1OdKnu#%u8pq4r09ia}$l@ zc@uTa;X-;b2xk@nD8ev9h77gMmyzC4d|6Kd#Zo}B5J~#}Gz}578!(n+)_xUdIZYE^ zA~H38eblz(xlMx=j-!+fVf%L$ne|#XY`KINWQxyuL?opKUmK=r)_Aw(c4FTwi4>J( zMtEYkGKB>#I%JVHSH+u%pl}27g-$i#n91LqZ4$Z+1C}JvKDjRY#2`s6zO922H<3lL zq(}}z8D_c>nc)|@h2P}A{~LMq(MO}CPYE&Z$CmQha1H4fA%9-nuQjRHOGvC5A3kAT>k8AN&o*Y{N^`3twSR?b|P3GI=Bj_JLNq3`G?lJ6tUkp5$dSjW z5S%7KP(JjYATx~q=+%9*?>m=kp$zo`*nzB9UHa%yin3A;Ga&Y|RCTx^sSJ0BTeOzo z^j46e)i@vuo4RmBPhn~l7ZJ3KZ3`mdR_=8m`vJI0gDvlPwPbaEG2_3;sn(jrhN-h0 z%MxcmMRdnU%r>);*mniMMWJ+rGf`Liy_mQ|^aS>Aa1l0+9|qkHlD@cNP(_An-5=D> za|x_m?3?UAw_`2&faFOTyh;?vR%!?PTGQ4pfsV$rI-rak)<}99N|{<6(^Rop69)q8 zm(2Bq-@%q2=3KT2^m!sF1%3`x%8NPN$}bP4J&E8jlMbeyNjNLkr&YcMHG`;k5;|;3 zX;K_vG?XGL!(N2(2(I>D=JJW-45n zw%lc8C**0e%raVQ0-jT_OnbQ(<&9nsko!8{UN`3x_~hDJ?A+uGsY~7Bzy9c+!X7{7 zjZ7Y;aWjme40Cw1vGHpC0 zACxh?%fQH6MWYABTN8yBNmW{j8D1?rX*KIDi-|z3xO!s+DcRCZdGhd8VVKf0lF}Qz z*m&u2xd-?TDidajyn*B?2@4Rr{kC_!qvGkoyxewwQ8_k5Mha3k-_ZJYH9>IHhQ$JJ z@dcv$lxE`0@f!-)FEn=e!)rTNz2Q96 z`a-b66UVG$3B4hm`f3ZvuAQ(~qcF5-e425dp$+8HB$!b8%1_GUZ^?&r!mZbtdLmj= zqU4*37t;sGQ$vxm{_Ws{2!hI<`xfJjLAbbYXcTTqM+vNQ3$;RdG1affLnFjoE5%Qo zsYlwK3e+WI1u^JPn~BoF{0|d}Jq@@j&7**Tsbd~nxTAN`vHC#_Tje2XkHqDWN)A^_Oxtc3RfvIhY1cV7CfEWiZ` zndF$M*|MafucV}$CWG?eGc9!lVHNnSvEymU$hdME)uj~q*?4OChL*jA>ea@;ao3&h zLN~BlJD@QM!q|~?S0Sg)*RE5#tSx*UnrM?}ioa+Bvt6}57vc|6{Q&&Mg9OyVJl6zF zyD@s1*Z9RuMO4}>zttv*aNUB~IadFCW7GIBNUEdP;hM*MNEJ;m_vHc>DFM*N3{@~8Gi?_B?P-=yco8C z7<5#~Q~FYYB#*9iKaML|K{y5~UR=wt*kMD^isPGI794wDR-DFECklAF4qSp>fxvR` zPVZWev5Wm~uDB#nZ!AIgEWP1hGIcC#9@8_PjwEU6;yqG zP?*Jy68P$QtPb%dzf1P>@ zt|6!{7D6)T&5W3eXH`>Ga`wEVS=N@@o2CIAGJschp*GoCMuiRvWlVbBqA)oIp)EzN zm9LIu%iE(6is?3O%gi1;*UT>Z6D8GhLIDs~mVmwF5U_8OEp(oPsS(t{`O5CX!r}JT zlupASXMG}E=`LHDmee4L?bYN6KM1)5UBx_p{e8^JtG}%O- zs#M@+3K7Pes(0=sZuX)Q@B^hy)tD+;?}Z*t4SnI$o8VjHsmtSBVzNpu9yWtaic$&E zKd>ZN;Un~d*-j}y1c+Lu*B#K=2%xB7jT+uZQQ^T~66}~&1w`_{e~;^ygm9h;3-~}& z5hGhb%|TV~vez>6Q#nFEM2G)P8c?Y)LhHZp8~;?qz>ID>yki3t6^MG@m0qd<1rE@7 z{9u9H|Ci1t2@rba)9J%@`@T(%0pbbMzlG4DoC{TzG^0TGy=tgt5dNpJ@pDaVcRZfP z_QNGdPYnbnW={UhTBg?-Xgtu%86t;Bzk{0TH^K86!j3tj083PS!syXAu0VJ2WJO;H<8b+5H`WYr zg|}l|Z%W^DL(Tx@sVT|Gi|yG9X`f8ri`oB49@1ZOOR?h#8g+dyc6;V}Twyh2%S4%I z2gJ0nO*2YeJhX=p9R@^5_byN#;SkTF^0r5z(`~)UO+EI)w(b6c?MV3hn+Y_!sve^g zp2vo$B8`H^C-|f`A0A~te#eO?O94q4Lil1~(gISr-9o*iU9|Xx1Hh$epp5|dapVB7 z?c>-3bsx*U5ft$O5~w&7ksnZrIDK-6?L*J`r_i4KM=G;NQ`>O!V$pp$nC`?IfRhBz7<;I* zmy4hSTP6BR4+eHxHy$E344s|t2nBLkWjms0o_RxP z@Tqn}nm6EC0zkC$3q_Ujk@ySmrFK?keBHlc{L4g2nzi~v?tLNM!Re?+D|{gSXI~Ve z7Mtgj&_8$C{7rOqq^T~-hS&f^)|b!G3|bXXkASDLHt-mK^ zX~KQe0t-xj_Mh%6f^5E16@-?OLz5 z&V0h0R)P?{LriDSEZk)o<&mq~nf?+)fxb(F>S*Hu*VNp2$%S((~Pde<)(- zqeD7{hmHGXA5TnRk%*t5@G3WUA_YL*x*w_%J|$hN;XoU^q6DSCt9JLqfppL^+%a?Wt=ngx%Y~RA@FTbJ*lGL^MSC9N}Gzyp*-b#}nr@ne)BvgA9FBa4irUBf1+J8&Etk9?23acryKA%7(! z(y**Z@xtLMRc-{s6zU$_0mNU(!e@2c&&su=Fa16o{W9(1iv~5L+c#_`;@I<&Iu)>s zG9EH+>MLI-3R#@C*8Po2cBLI6Q4yG?ZPq_Ie^%a@Q*tc#p;TIGM5YqY{NO)6hn!tx zRG_Fd%J1nA+&)tfH`AglMIs zO(B}r(3P`;v3xnHJsPEM zRC(r>EtO_UbQEbTlj5*&**Z$OFwVy9#(CT^9#=rfDn!G+?7wM%$jGP{iG_3NH zQWK`Y9m#dmY5bG78`EtMHKKR_x=84_ZHy_Ls$w+@gr_^-5eph!v?N;^$XqQQ-Sy5I zujV4RDAB*|^*Gr8EbcsAqZ%K$-{>~Ne4`xCyPHgbh!cXQtnA^;iDi5Mnl#Hgz z6PlDlhEBS6(Z`c8eTd>Ral5~(z*^u~egA@S!;F)7+OJUUweA;;Z1pKcSV}**1CMEC z3RIxJeti>?v4n(Qsg*#ZepWWOC@t!i&&)0>PdS&0&0YBcw`l_)yRtcAcCVZSZZ*-J zwQbkrX$Jh>ajWG34c?Tnb&fS(I?C-03=HOQx8pyb@@OA`e%cXxbzXtMA&No$wyBEN z-kmb*UbZ2+_M#CUf8_OnG-M_U5nAJETa@5`@UayZs55Dyk|H&T)?}=v>o4FZ{&20@ zN`}hV{H52eFh)O&gN&*#T>Az^Kat6TgZhd4Uz^tyd+`e!kK4YY_8UdC8`ne}gk!%d zpSJ9|uKI?t@qZ#rH}S7A>Pz_709tiZaLHr+2d>*ZY7$kpA$F5ZpIi&~)@=@3UBdkO zEeQ7PV+m)Zo65}B03d9g*mX!KpZZKqItNtBHZwxbMza~FsXPZLAAR<$y&$!6SoJZMIGAIU*M70^=J`tz+-%H-HhRZCAF#FgzWPf)No7s+U*#f@;9-- z7FVTq-FKb<3Q%WUWS|6lvQJywak6w&r5D)Z69i!bg(Oc95vg7$Aw=^85rvqivQ#&K zEl#4ac(dR_4Sz!~NE2Ej)e}g%2GP#@P*nZiyR@h$u=eSHI79!u`C|o~soFV&uJ?Q# zFkmgU*8a!hrva^)%nxke0RzrE>6B`e0>!D&;>hHhI!<6fn?chW)f!4*z*0fz)YAXY d&Q>1LyFlL&tW)EYKus} + + 4.0.0 + + qunar.tc + qmq + 4.0.30 + pom + + 2012 + + + 3.2.5 + + + + + Apache License, Version 2.0 + http://www.apache.org/licenses/LICENSE-2.0 + repo + + + + + Qunar Inc. + http://www.qunar.com + + + + qmq-client + qmq-api + qmq-common + qmq-remoting + qmq-store + qmq-metaserver + qmq-server-common + qmq-delay-server + qmq-server + qmq-sync + qmq-metrics-prometheus + qmq-dist + qmq-tools + + + + 1.8 + 1.8 + + 0.5.0 + 4.2.3.RELEASE + + + + + + ${project.groupId} + qmq-api + ${project.version} + + + ${project.groupId} + qmq-common + ${project.version} + + + ${project.groupId} + qmq-remoting + ${project.version} + + + ${project.groupId} + qmq-client + ${project.version} + + + ${project.groupId} + qmq-server-common + ${project.version} + + + ${project.groupId} + qmq-store + ${project.version} + + + ${project.groupId} + qmq-sync + ${project.version} + + + ${project.groupId} + qmq-metrics-prometheus + ${project.version} + + + qunar.tc + qmq-metaserver + ${project.version} + + + qunar.tc + qmq-server + ${project.version} + + + qunar.tc + qmq-delay-server + ${project.version} + + + qunar.tc + qmq-tools + ${project.version} + + + + javax.servlet + javax.servlet-api + 3.1.0 + + + + mysql + mysql-connector-java + 5.1.39 + + + + com.zaxxer + HikariCP + 2.6.2 + + + io.prometheus + simpleclient + ${prometheus.client.version} + + + io.opentracing + opentracing-util + 0.31.0 + + + org.springframework + spring-context + ${spring.version} + + + org.springframework + spring-jdbc + ${spring.version} + + + + com.google.guava + guava + 23.0 + + + io.netty + netty-all + 4.0.24.Final + + + + org.slf4j + slf4j-api + 1.7.25 + + + org.slf4j + jcl-over-slf4j + 1.7.25 + + + org.slf4j + log4j-over-slf4j + 1.7.25 + + + ch.qos.logback + logback-classic + 1.2.3 + + + ch.qos.logback + logback-core + 1.2.3 + + + + com.fasterxml.jackson.core + jackson-core + 2.9.6 + + + com.fasterxml.jackson.core + jackson-databind + 2.9.6 + + + com.fasterxml.jackson.core + jackson-annotations + 2.9.6 + + + + org.eclipse.jetty + jetty-server + 9.4.14.v20181114 + + + org.eclipse.jetty + jetty-servlet + 9.4.14.v20181114 + + + + joda-time + joda-time + 2.9.9 + + + + com.ning + async-http-client + 1.9.40 + + + info.picocli + picocli + 3.8.2 + + + + junit + junit + 4.12 + test + + + + + + + + + maven-compiler-plugin + 3.5.1 + + ${java_source_version} + ${java_target_version} + UTF-8 + true + true + + + + maven-source-plugin + 3.0.1 + + + attach-sources + + jar + + + + + + org.apache.maven.plugins + maven-resources-plugin + 2.5 + + UTF-8 + false + \ + + ${*} + + + + + maven-assembly-plugin + 3.0.0 + + + + + \ No newline at end of file diff --git a/qmq-api/pom.xml b/qmq-api/pom.xml new file mode 100644 index 00000000..0521e4e6 --- /dev/null +++ b/qmq-api/pom.xml @@ -0,0 +1,19 @@ + + + 4.0.0 + + + qmq + qunar.tc + 4.0.30 + + + qmq-api + + + 1.7 + 1.7 + + \ No newline at end of file diff --git a/qmq-api/src/main/java/qunar/tc/qmq/BaseConsumer.java b/qmq-api/src/main/java/qunar/tc/qmq/BaseConsumer.java new file mode 100644 index 00000000..1371add8 --- /dev/null +++ b/qmq-api/src/main/java/qunar/tc/qmq/BaseConsumer.java @@ -0,0 +1,27 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq; + +/** + * @author yiqun.fan create on 17-9-12. + */ +public interface BaseConsumer { + + String subject(); + + String group(); +} diff --git a/qmq-api/src/main/java/qunar/tc/qmq/Filter.java b/qmq-api/src/main/java/qunar/tc/qmq/Filter.java new file mode 100644 index 00000000..756a4f2c --- /dev/null +++ b/qmq-api/src/main/java/qunar/tc/qmq/Filter.java @@ -0,0 +1,45 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq; + +import java.util.Map; + +/** + * Created by zhaohui.yu + * 15/12/7 + */ +public interface Filter { + + /** + * 在listener.onMessage之前执行 + * + * @param message 处理的消息,建议不要修改消息内容 + * @param filterContext 可以在这里保存一些上下文 + * @return 如果返回true则filter链继续往下执行,只要任一filter返回false,则后续的 + * filter链不会执行,并且listener.onMessage也不会执行 + */ + boolean preOnMessage(Message message, Map filterContext); + + /** + * 在listener.onMessage之后执行,可以做一些资源清理工作 + * + * @param message 处理的消息 + * @param e filter链和listener.onMessage抛出的异常 + * @param filterContext 上下文 + */ + void postOnMessage(Message message, Throwable e, Map filterContext); +} diff --git a/qmq-api/src/main/java/qunar/tc/qmq/FilterAttachable.java b/qmq-api/src/main/java/qunar/tc/qmq/FilterAttachable.java new file mode 100644 index 00000000..17bf3461 --- /dev/null +++ b/qmq-api/src/main/java/qunar/tc/qmq/FilterAttachable.java @@ -0,0 +1,29 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq; + +import java.util.List; + +/** + * Created by zhaohui.yu + * 15/12/8 + * + * MessageListener如果实现了这个接口,则可以附加filter + */ +public interface FilterAttachable { + List filters(); +} diff --git a/qmq-api/src/main/java/qunar/tc/qmq/IdempotentAttachable.java b/qmq-api/src/main/java/qunar/tc/qmq/IdempotentAttachable.java new file mode 100644 index 00000000..0bfce452 --- /dev/null +++ b/qmq-api/src/main/java/qunar/tc/qmq/IdempotentAttachable.java @@ -0,0 +1,27 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq; + +/** + * Created by zhaohui.yu + * 15/12/8 + *

+ * 通过这个接口可以给MessageListener添加幂等检查的功能 + */ +public interface IdempotentAttachable { + IdempotentChecker getIdempotentChecker(); +} diff --git a/qmq-api/src/main/java/qunar/tc/qmq/IdempotentChecker.java b/qmq-api/src/main/java/qunar/tc/qmq/IdempotentChecker.java new file mode 100644 index 00000000..08fa18e9 --- /dev/null +++ b/qmq-api/src/main/java/qunar/tc/qmq/IdempotentChecker.java @@ -0,0 +1,51 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq; + +/** + * Created by zhaohui.yu + * 15/11/4 + *

+ * 幂等检查 + *

+ * 已经提供了 + * + * @see qunar.tc.qmq.consumer.idempotent.JdbcIdempotentChecker + * @see qunar.tc.qmq.consumer.idempotent.TransactionalJdbcIdempotentChecker + *

+ * 如果不能满足需求,最好从 + * @see qunar.tc.qmq.consumer.idempotent.AbstractIdempotentChecker + * 派生 + */ +public interface IdempotentChecker { + + /** + * 消息是否已经处理过 + * + * @param message 投递过来的消息 + * @return 是否已经处理 + */ + boolean isProcessed(Message message); + + /** + * 标记消息是否处理 + * + * @param message 消息 + * @param e 消费消息是否出异常 + */ + void markProcessed(Message message, Throwable e); +} diff --git a/qmq-api/src/main/java/qunar/tc/qmq/ListenerHolder.java b/qmq-api/src/main/java/qunar/tc/qmq/ListenerHolder.java new file mode 100644 index 00000000..157e7a05 --- /dev/null +++ b/qmq-api/src/main/java/qunar/tc/qmq/ListenerHolder.java @@ -0,0 +1,26 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ +package qunar.tc.qmq; + +/** + * @author miao.yang susing@gmail.com + * @date 2012-12-26 + */ +public interface ListenerHolder { + void stopListen(); + + void resumeListen(); +} diff --git a/qmq-api/src/main/java/qunar/tc/qmq/Message.java b/qmq-api/src/main/java/qunar/tc/qmq/Message.java new file mode 100644 index 00000000..dc1785ca --- /dev/null +++ b/qmq-api/src/main/java/qunar/tc/qmq/Message.java @@ -0,0 +1,173 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq; + +import java.util.Date; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.TimeUnit; + +/** + * @author miao.yang susing@gmail.com + * @date 2012-12-26 + */ +public interface Message { + + /** + * 获取message id,一般是qmq自动生成的,message id需要保证全局唯一,不要将message id用于业务,一般仅做记录日志 + * + * @return message id + */ + String getMessageId(); + + /** + * 消息的主题 + * + * @return subject + */ + String getSubject(); + + /** + * 消息的创建时间,指的是调用generateMessage方法的时间 + * + * @return + */ + Date getCreatedTime(); + + Date getScheduleReceiveTime(); + + void setProperty(String name, boolean value); + + void setProperty(String name, Boolean value); + + void setProperty(String name, int value); + + void setProperty(String name, Integer value); + + void setProperty(String name, long value); + + void setProperty(String name, Long value); + + void setProperty(String name, float value); + + void setProperty(String name, Float value); + + void setProperty(String name, double value); + + void setProperty(String name, Double value); + + void setProperty(String name, Date date); + + void setProperty(String name, String value); + + /** + * 可以设置4MB的超大字符串,但是要注意,使用这个方法设置的字符串必须使用getLargeString方法获取 + * 另外,超大消息不提供持久化支持,不能使用事务或持久消息 + * + * @param name key + * @param value value + */ + void setLargeString(String name, String value); + + String getStringProperty(String name); + + boolean getBooleanProperty(String name); + + Date getDateProperty(String name); + + int getIntProperty(String name); + + long getLongProperty(String name); + + float getFloatProperty(String name); + + double getDoubleProperty(String name); + + /** + * 获取setLargeString方法设置的字符串 + * + * @param name key + * @return value + */ + String getLargeString(String name); + + /** + * 这个方法不是线程安全 + * + * @param tag 不能是null或empty, 长度不能超过Short.MAX_VALUE,tag的个数最多不能超过10个 + * @return this + */ + Message addTag(String tag); + + /** + * @return 默认返回empty set, 返回的是不可变set + */ + Set getTags(); + + /** + * @return + * @deprecated 禁止使用该方法 + */ + @Deprecated + Map getAttrs(); + + /** + * 期望显式的手动ack时,使用该方法关闭qmq默认的自动ack。 + * 该方法必须是在consumer端的MessageListener的onMessage方法入口处调用,否则会抛出异常 + *

+ * 在producer端调用时会抛出UnsupportedOperationException异常 + * + * @param auto + */ + void autoAck(boolean auto); + + /** + * 显式手动ack的时候,使用该方法 + * + * @param elapsed 消息处理时长 + * @param e 如果消息处理失败请传入异常,否则传null + *

+ * 在producer端调用会抛出UnsupportedOperationException异常 + */ + void ack(long elapsed, Throwable e); + + void ack(long elapsed, Throwable e, Map attachment); + + + void setDelayTime(Date date); + + void setDelayTime(long delayTime, TimeUnit timeUnit); + + /** + * 第几次发送 + * 使用方应该监控该次数,如果不是刻意设计该次数不应该太多 + * + * @return + */ + int times(); + + void setMaxRetryNum(int maxRetryNum); + + int getMaxRetryNum(); + + /** + * 本地连续重试次数 + * + * @return + */ + int localRetries(); +} diff --git a/qmq-api/src/main/java/qunar/tc/qmq/MessageConsumer.java b/qmq-api/src/main/java/qunar/tc/qmq/MessageConsumer.java new file mode 100644 index 00000000..3adb578e --- /dev/null +++ b/qmq-api/src/main/java/qunar/tc/qmq/MessageConsumer.java @@ -0,0 +1,52 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ +package qunar.tc.qmq; + +import java.util.concurrent.Executor; + +public interface MessageConsumer { + + /** + * 注册消息处理程序 + * + * @param subject 订阅的消息主题 + * @param consumerGroup consumer分组,用于consumer的负载均衡(broker只会给每个consumer group发送一条消息)。 + * 如果想每个consumer进程都收到消息(广播模式),只需要给group参数传空字符串即可。 + * @param listener 消息处理程序 + * @param executor 消息处理线程池 + * @return 返回的ListenerHolder, 表示注册关系 + * @deprecated 标记为过期,请使用不需要传递连接池的方法 + */ + ListenerHolder addListener(String subject, String consumerGroup, MessageListener listener, Executor executor); + + /** + * 注册消息处理程序 + * + * @param subject 订阅的消息主题 + * @param consumerGroup consumer分组,用于consumer的负载均衡(broker只会给每个consumer group发送一条消息)。 + * 如果想每个consumer进程都收到消息(广播模式),只需要给group参数传空字符串即可。 + * @param listener 消息处理程序 + * @param executor 消息处理线程池 + * @return 返回的ListenerHolder, 表示注册关系 + */ + ListenerHolder addListener(String subject, String consumerGroup, MessageListener listener, Executor executor, SubscribeParam subscribeParam); + + /** + * @param group nullOrEmpty时,是广播订阅 + * @param isBroadcast 等于true时,忽略group参数,广播订阅;等于false时,group不能是nullOrEmpty + */ + PullConsumer getOrCreatePullConsumer(String subject, String group, boolean isBroadcast); +} diff --git a/qmq-api/src/main/java/qunar/tc/qmq/MessageListener.java b/qmq-api/src/main/java/qunar/tc/qmq/MessageListener.java new file mode 100644 index 00000000..2ce5510f --- /dev/null +++ b/qmq-api/src/main/java/qunar/tc/qmq/MessageListener.java @@ -0,0 +1,26 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ +package qunar.tc.qmq; + +/** + * @author miao.yang susing@gmail.com + * @date 2012-12-26 + */ +public interface MessageListener { + + void onMessage(Message msg); + +} diff --git a/qmq-api/src/main/java/qunar/tc/qmq/MessageProducer.java b/qmq-api/src/main/java/qunar/tc/qmq/MessageProducer.java new file mode 100644 index 00000000..4c617f15 --- /dev/null +++ b/qmq-api/src/main/java/qunar/tc/qmq/MessageProducer.java @@ -0,0 +1,47 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ +package qunar.tc.qmq; + +/** + * @author miao.yang susing@gmail.com + * @date 2012-12-26 + */ +public interface MessageProducer { + + /** + * 在发送消息之前调用该接口生成消息,该接口会生成唯一消息id。这条消息的过期时间为默认的15分钟 + * + * @param subject 要发送的消息的subject + * @return 生成的消息 + */ + Message generateMessage(String subject); + + /** + * 发送消息 + * 注意:在使用事务性消息时该方法仅将消息入库,当事务成功提交时才发送消息。只要事务提交,消息就会发送(即使入库失败)。 + * 在使用事务消息时,该方法会使用业务方配置的数据源操作qmq_produce数据库,请确保为业务datasource配置的数据库用户 + * 具有对qmq_produce数据库的CURD权限。 + *

+ * 该方法须在generateMessage方法调用之后调用 @see generateMessage + * + * @param message + * @throws RuntimeException 当消息体超过指定大小(60K)的时候会抛出RuntimeException异常 + * 当消息过期时间设置非法的时候会抛出RuntimeException异常 + */ + void sendMessage(Message message); + + void sendMessage(Message message, MessageSendStateListener listener); +} diff --git a/qmq-api/src/main/java/qunar/tc/qmq/MessageSendStateListener.java b/qmq-api/src/main/java/qunar/tc/qmq/MessageSendStateListener.java new file mode 100644 index 00000000..aa95b819 --- /dev/null +++ b/qmq-api/src/main/java/qunar/tc/qmq/MessageSendStateListener.java @@ -0,0 +1,40 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq; + +/** + * User: zhaohuiyu + * Date: 5/24/13 + * Time: 3:31 PM + *

+ * 消息的发送状态监听器 + */ +public interface MessageSendStateListener { + /** + * 消息成功发送后触发 + * + * @param message + */ + void onSuccess(Message message); + + /** + * 消息发送失败时触发 + * + * @param message + */ + void onFailed(Message message); +} diff --git a/qmq-api/src/main/java/qunar/tc/qmq/NeedRetryException.java b/qmq-api/src/main/java/qunar/tc/qmq/NeedRetryException.java new file mode 100644 index 00000000..caac1d50 --- /dev/null +++ b/qmq-api/src/main/java/qunar/tc/qmq/NeedRetryException.java @@ -0,0 +1,54 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq; + +import java.util.Date; +import java.util.concurrent.TimeUnit; + +/** + * Created by zhaohui.yu + * 15/12/2 + *

+ * qmq会根据该异常里的时间进行重试间隔控制 + */ +public class NeedRetryException extends RuntimeException { + private final long next; + + public NeedRetryException(Date next, String message) { + super(message); + this.next = next.getTime(); + } + + public NeedRetryException(int next, TimeUnit unit, String message) { + super(message); + this.next = System.currentTimeMillis() + unit.toMillis(next); + } + + /** + * WARNING WARNING + * 使用该构造函数构造的异常会立即重试 + * + * @param message + */ + public NeedRetryException(String message) { + this(new Date(), message); + } + + public long getNext() { + return next; + } +} diff --git a/qmq-api/src/main/java/qunar/tc/qmq/ProduceMessage.java b/qmq-api/src/main/java/qunar/tc/qmq/ProduceMessage.java new file mode 100644 index 00000000..3ef3f86e --- /dev/null +++ b/qmq-api/src/main/java/qunar/tc/qmq/ProduceMessage.java @@ -0,0 +1,42 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq; + +/** + * Created by zhaohui.yu + * 10/31/16 + */ +public interface ProduceMessage { + + String getMessageId(); + + String getSubject(); + + void send(); + + void error(Exception e); + + void failed(); + + void block(); + + void finish(); + + Message getBase(); + + void startSendTrace(); +} diff --git a/qmq-api/src/main/java/qunar/tc/qmq/PullConsumer.java b/qmq-api/src/main/java/qunar/tc/qmq/PullConsumer.java new file mode 100644 index 00000000..5e560631 --- /dev/null +++ b/qmq-api/src/main/java/qunar/tc/qmq/PullConsumer.java @@ -0,0 +1,100 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq; + +import java.util.List; +import java.util.concurrent.Future; + +/** + * @author yiqun.fan create on 17-9-11. + */ +public interface PullConsumer extends BaseConsumer, AutoCloseable { + + void online(); + + void offline(); + + /** + * 设置true时,每条消息(包括可靠)只消费一次,拉取后自动ack,无需调用ack方法。 + * 设置false时,可靠消息不会自动ack,需要调用ack方法,非可靠不需要。 + * 默认值是false。 + */ + void setConsumeMostOnce(boolean consumeMostOnce); + + /** + * 返回是否设置了consumeMostOnce + */ + boolean isConsumeMostOnce(); + + /** + * 参考 qunar.tc.qmq.consumer.annotation.QmqConsumer#filterTags() + */ + // TODO consumer + //void setFilterTags(String[] filterTags); + + // TODO consumer + //String[] filterTags(); + + /** + * 拉取到size个消息后才返回。 + * 如果producer发送的消息个数没有达到size,则pull方法会被阻塞住。 + * 如果当前拉取线程被Interrupted, 则返回已经拉取到的消息。 + * 返回的消息个数可能不等于size。 + *

+ * 对于可靠消息处理完成后,必须调用Message的ack方法,然后再次拉取。 + * 对于非可靠消息,无需调用ack方法。 + */ + List pull(int size); + + /** + * 尝试在timeoutMillis内拉取size个消息。 + * 返回的消息个数可能不等于size。 + * 如果size <= 0,会立即返回一个空的List。 + *

+ * 当broker上剩余的消息数小于size时,pull方法会阻塞至超时。 + * 实际调用时间可能大于timeoutMillis。 + * timeoutMillis最小值是1000,应尽量设置大的timeout。 + * + * 如果timeoutMillis设置为小于0的时候,则表示不管队列里有没有消息都拉一下立即返回, + * 这样实际返回的消息条数可能小于size,但是不会大于size + *

+ * 对于可靠消息处理完成后,必须调用Message的ack方法,然后再次拉取。 + * 对于非可靠消息,无需调用ack方法。 + */ + List pull(int size, long timeoutMillis); + + /** + * 返回的Future不支持cancel()方法,调用cancel()会抛出UnsupportedOperationException异常。 + * 如果调用Future的get方法发生超时,必须在之后调用get()直到isDone()返回true。 + *

+ * 对于可靠消息处理完成后,必须调用Message的ack方法,然后再次拉取。 + * 对于非可靠消息,无需调用ack方法。 + */ + Future> pullFuture(int size); + + /** + * 实际的拉取时间可能超过timeoutMillis,所以必须等到isDone()返回true才可以丢弃返回的future, + * 否则可能出现拉取到的消息没有被获取到。 + * timeoutMillis的设置参考 qunar.tc.qmq.PullConsumer#pull(int, long) 的注释 + *

+ * 对于可靠消息处理完成后,必须调用Message的ack方法,然后再次拉取。 + * 对于非可靠消息,无需调用ack方法。 + */ + Future> pullFuture(int size, long timeoutMillis); + + Future> pullFuture(int size, long timeoutMillis, boolean isResetCreateTime); +} diff --git a/qmq-api/src/main/java/qunar/tc/qmq/SubscribeParam.java b/qmq-api/src/main/java/qunar/tc/qmq/SubscribeParam.java new file mode 100644 index 00000000..18ecc769 --- /dev/null +++ b/qmq-api/src/main/java/qunar/tc/qmq/SubscribeParam.java @@ -0,0 +1,94 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq; + +import java.util.Collections; +import java.util.Set; + +/** + * @author yiqun.fan create on 17-11-2. + */ +public class SubscribeParam { + public static final SubscribeParam DEFAULT = new SubscribeParam(false, false, TagType.NO_TAG, Collections.emptySet()); + + private final boolean consumeMostOnce; + private final TagType tagType; + private boolean isBroadcast; + private final Set tags; + + private SubscribeParam(boolean consumeMostOnce, boolean isBroadcast, TagType tagType, Set tags) { + this.consumeMostOnce = consumeMostOnce; + this.isBroadcast = isBroadcast; + this.tags = tags; + this.tagType = tagType; + } + + public boolean isConsumeMostOnce() { + return consumeMostOnce; + } + + public Set getTags() { + return tags; + } + + public TagType getTagType() { + return tagType; + } + + public boolean isBroadcast() { + return isBroadcast; + } + + public void setBroadcast(boolean isBroadcast) { + this.isBroadcast = isBroadcast; + } + + public static final class SubscribeParamBuilder { + private boolean consumeMostOnce = false; + private Set tags = Collections.emptySet(); + private TagType tagType = TagType.NO_TAG; + private boolean isBroadcast; + + public SubscribeParam create() { + return new SubscribeParam(consumeMostOnce, isBroadcast, tagType, tags); + } + + public SubscribeParamBuilder setConsumeMostOnce(boolean consumeMostOnce) { + this.consumeMostOnce = consumeMostOnce; + return this; + } + + public SubscribeParamBuilder setTagType(final TagType tagType) { + if (tagType != null) { + this.tagType = tagType; + } + return this; + } + + public SubscribeParamBuilder setTags(Set tags) { + if (tags != null && tags.size() != 0) { + this.tags = tags; + } + return this; + } + + public SubscribeParamBuilder setBroadcast(boolean isBroadcast) { + this.isBroadcast = isBroadcast; + return this; + } + } +} diff --git a/qmq-api/src/main/java/qunar/tc/qmq/TagType.java b/qmq-api/src/main/java/qunar/tc/qmq/TagType.java new file mode 100644 index 00000000..ad869ccb --- /dev/null +++ b/qmq-api/src/main/java/qunar/tc/qmq/TagType.java @@ -0,0 +1,54 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq; + +import java.util.HashMap; +import java.util.Map; + +/** + * @author yunfeng.yang + * @since 2018/1/18 + */ +public enum TagType { + NO_TAG(0), OR(1), AND(2); + + private int code; + + TagType(int code) { + this.code = code; + } + + private final static Map MAP = new HashMap<>(); + + static { + for (TagType tagType : TagType.values()) { + MAP.put(tagType.getCode(), tagType); + } + } + + public static TagType of(final int code) { + final TagType tagType = MAP.get(code); + if (tagType != null) { + return tagType; + } + return TagType.NO_TAG; + } + + public int getCode() { + return code; + } +} diff --git a/qmq-client/pom.xml b/qmq-client/pom.xml new file mode 100644 index 00000000..d3769475 --- /dev/null +++ b/qmq-client/pom.xml @@ -0,0 +1,60 @@ + + + 4.0.0 + + + qmq + qunar.tc + 4.0.30 + + + qmq-client + + + 1.7 + 1.7 + + + + + ${project.groupId} + qmq-common + + + ${project.groupId} + qmq-api + + + ${project.groupId} + qmq-remoting + + + org.springframework + spring-context + true + + + org.springframework + spring-jdbc + true + + + com.google.guava + guava + + + io.netty + netty-all + + + io.opentracing + opentracing-util + + + org.slf4j + slf4j-api + + + \ No newline at end of file diff --git a/qmq-client/src/main/java/qunar/tc/qmq/broker/BrokerClusterInfo.java b/qmq-client/src/main/java/qunar/tc/qmq/broker/BrokerClusterInfo.java new file mode 100644 index 00000000..2dcdf4b6 --- /dev/null +++ b/qmq-client/src/main/java/qunar/tc/qmq/broker/BrokerClusterInfo.java @@ -0,0 +1,57 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.broker; + +import com.google.common.collect.Maps; + +import java.util.Collections; +import java.util.List; +import java.util.Map; + +/** + * @author yiqun.fan create on 17-8-18. + */ +public class BrokerClusterInfo { + private final List groupList; + private final Map groupMap; + + public BrokerClusterInfo() { + this.groupList = Collections.emptyList(); + this.groupMap = Collections.emptyMap(); + } + + public BrokerClusterInfo(List groupList) { + this.groupList = groupList; + this.groupMap = Maps.newHashMapWithExpectedSize(groupList.size()); + for (BrokerGroupInfo group : groupList) { + groupMap.put(group.getGroupName(), group); + } + } + + public List getGroups() { + return groupList; + } + + public BrokerGroupInfo getGroupByName(String groupName) { + return groupMap.get(groupName); + } + + @Override + public String toString() { + return "BrokerClusterInfo{groups=" + groupList + "}"; + } +} diff --git a/qmq-client/src/main/java/qunar/tc/qmq/broker/BrokerGroupInfo.java b/qmq-client/src/main/java/qunar/tc/qmq/broker/BrokerGroupInfo.java new file mode 100644 index 00000000..949212bd --- /dev/null +++ b/qmq-client/src/main/java/qunar/tc/qmq/broker/BrokerGroupInfo.java @@ -0,0 +1,99 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.broker; + +import com.google.common.base.Preconditions; +import com.google.common.base.Strings; + +import java.util.List; + +/** + * @author yiqun.fan create on 17-8-18. + */ +public class BrokerGroupInfo { + + private final int groupIndex; + private final String groupName; + private final String master; + private final List slaves; + private volatile boolean available = true; + + private final CircuitBreaker circuitBreaker; + + public BrokerGroupInfo(int groupIndex, String groupName, String master, List slaves) { + Preconditions.checkArgument(!Strings.isNullOrEmpty(groupName), "groupName不能是空"); + Preconditions.checkArgument(!Strings.isNullOrEmpty(master), "master不能是空"); + this.groupIndex = groupIndex; + this.groupName = groupName; + this.master = master; + this.slaves = slaves; + this.circuitBreaker = new CircuitBreaker(); + } + + public int getGroupIndex() { + return groupIndex; + } + + public String getGroupName() { + return groupName; + } + + public String getMaster() { + return master; + } + + public List getSlaves() { + return slaves; + } + + public void setAvailable(boolean available) { + this.available = available; + } + + public boolean isAvailable() { + return circuitBreaker.isAvailable() && available; + } + + public void markFailed() { + circuitBreaker.markFailed(); + } + + public void markSuccess() { + circuitBreaker.markSuccess(); + } + + public static boolean isInvalid(BrokerGroupInfo brokerGroup) { + return brokerGroup == null || !brokerGroup.isAvailable(); + } + + @Override + public String toString() { + return "BrokerGroupInfo{group=" + groupName + ", " + + "master=" + master + ", " + + "slaves=" + slaves + "}"; + } + + @Override + public boolean equals(Object obj) { + return obj == this || (obj instanceof BrokerGroupInfo && groupName.equals(((BrokerGroupInfo) obj).groupName)); + } + + @Override + public int hashCode() { + return groupName.hashCode(); + } +} diff --git a/qmq-client/src/main/java/qunar/tc/qmq/broker/BrokerLoadBalance.java b/qmq-client/src/main/java/qunar/tc/qmq/broker/BrokerLoadBalance.java new file mode 100644 index 00000000..65149d75 --- /dev/null +++ b/qmq-client/src/main/java/qunar/tc/qmq/broker/BrokerLoadBalance.java @@ -0,0 +1,24 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.broker; + +/** + * @author yiqun.fan create on 17-8-18. + */ +public interface BrokerLoadBalance { + BrokerGroupInfo loadBalance(BrokerClusterInfo cluster, BrokerGroupInfo lastGroup); +} diff --git a/qmq-client/src/main/java/qunar/tc/qmq/broker/BrokerService.java b/qmq-client/src/main/java/qunar/tc/qmq/broker/BrokerService.java new file mode 100644 index 00000000..42a52ef8 --- /dev/null +++ b/qmq-client/src/main/java/qunar/tc/qmq/broker/BrokerService.java @@ -0,0 +1,35 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.broker; + +import qunar.tc.qmq.common.ClientType; + +/** + * @author yiqun.fan create on 17-8-18. + */ +public interface BrokerService { + + BrokerClusterInfo getClusterBySubject(ClientType clientType, String subject); + + BrokerClusterInfo getClusterBySubject(ClientType clientType, String subject, String group); + + void refresh(ClientType clientType, String subject); + + void refresh(ClientType clientType, String subject, String group); + + void setAppCode(String appCode); +} diff --git a/qmq-client/src/main/java/qunar/tc/qmq/broker/CircuitBreaker.java b/qmq-client/src/main/java/qunar/tc/qmq/broker/CircuitBreaker.java new file mode 100644 index 00000000..ed390c4f --- /dev/null +++ b/qmq-client/src/main/java/qunar/tc/qmq/broker/CircuitBreaker.java @@ -0,0 +1,153 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.broker; + +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; + +/** + * Created by zhaohui.yu + * 5/31/18 + */ +class CircuitBreaker { + + private final AtomicReference state; + + CircuitBreaker() { + state = new AtomicReference(new Closed(this)); + } + + private void switchState(State current, State to) { + state.compareAndSet(current, to); + } + + private void switchState(State to) { + state.set(to); + } + + void markFailed() { + state.get().markFailed(); + } + + void markSuccess() { + state.get().markSuccess(); + } + + boolean isAvailable() { + return state.get().isAvailable(); + } + + private interface State { + + void markFailed(); + + void markSuccess(); + + boolean isAvailable(); + } + + private static class Open implements State { + + private final CircuitBreaker circuitBreaker; + + private final long lastFailedTs; + + Open(CircuitBreaker circuitBreaker) { + this.circuitBreaker = circuitBreaker; + this.lastFailedTs = System.currentTimeMillis(); + } + + @Override + public void markFailed() { + + } + + @Override + public void markSuccess() { + + } + + @Override + public boolean isAvailable() { + if (System.currentTimeMillis() - lastFailedTs > 5000) { + circuitBreaker.switchState(this, new HalfOpen(circuitBreaker)); + return true; + } + return false; + } + } + + private static class HalfOpen implements State { + private final AtomicInteger successCount; + private final CircuitBreaker circuitBreaker; + + private final int maxSuccessCount = 20; + + + HalfOpen(CircuitBreaker circuitBreaker) { + this.circuitBreaker = circuitBreaker; + this.successCount = new AtomicInteger(0); + } + + @Override + public boolean isAvailable() { + return true; + } + + @Override + public void markFailed() { + successCount.set(0); + circuitBreaker.switchState(this, new Open(circuitBreaker)); + } + + public void markSuccess() { + if (successCount.incrementAndGet() >= maxSuccessCount) { + circuitBreaker.switchState(new Closed(circuitBreaker)); + } + } + } + + private static class Closed implements State { + + private final AtomicInteger failed; + private final CircuitBreaker circuitBreaker; + private final int maxFailedCount = 100; + + Closed(CircuitBreaker circuitBreaker) { + this.circuitBreaker = circuitBreaker; + this.failed = new AtomicInteger(0); + } + + @Override + public void markFailed() { + if (failed.incrementAndGet() >= maxFailedCount) { + circuitBreaker.switchState(this, new Open(circuitBreaker)); + } + } + + @Override + public void markSuccess() { + failed.set(0); + } + + @Override + public boolean isAvailable() { + return true; + } + + } +} diff --git a/qmq-client/src/main/java/qunar/tc/qmq/broker/impl/BrokerServiceImpl.java b/qmq-client/src/main/java/qunar/tc/qmq/broker/impl/BrokerServiceImpl.java new file mode 100644 index 00000000..db38879d --- /dev/null +++ b/qmq-client/src/main/java/qunar/tc/qmq/broker/impl/BrokerServiceImpl.java @@ -0,0 +1,153 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.broker.impl; + +import com.google.common.eventbus.Subscribe; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import qunar.tc.qmq.broker.BrokerClusterInfo; +import qunar.tc.qmq.broker.BrokerService; +import qunar.tc.qmq.common.ClientType; +import qunar.tc.qmq.common.MapKeyBuilder; +import qunar.tc.qmq.metainfoclient.MetaInfo; +import qunar.tc.qmq.metainfoclient.MetaInfoService; + +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicReference; + +/** + * @author yiqun.fan create on 17-8-18. + */ +public class BrokerServiceImpl implements BrokerService { + private static final Logger LOGGER = LoggerFactory.getLogger(BrokerServiceImpl.class); + + private final ConcurrentMap clusterMap = new ConcurrentHashMap<>(); + + private final MetaInfoService metaInfoService; + private String appCode; + + public BrokerServiceImpl(MetaInfoService metaInfoService) { + this.metaInfoService = metaInfoService; + this.metaInfoService.register(this); + } + + @Subscribe + public void onReceiveMetaInfo(MetaInfo metaInfo) { + LOGGER.debug("BrokerServiceOnReceiveMetaInfo", "receive MetaInfo: {}", metaInfo); + String key = MapKeyBuilder.buildMetaInfoKey(metaInfo.getClientType(), metaInfo.getSubject()); + ClusterFuture future = clusterMap.get(key); + if (future == null) { + future = new ClusterFuture(metaInfo.getClusterInfo()); + ClusterFuture oldFuture = clusterMap.putIfAbsent(key, future); + if (oldFuture != null) { + oldFuture.set(metaInfo.getClusterInfo()); + } + } else { + future.set(metaInfo.getClusterInfo()); + } + } + + @Override + public BrokerClusterInfo getClusterBySubject(ClientType clientType, String subject) { + return getClusterBySubject(clientType, subject, ""); + } + + @Override + public BrokerClusterInfo getClusterBySubject(ClientType clientType, String subject, String group) { + // 这个key上加group不兼容MetaInfoResponse + String key = MapKeyBuilder.buildMetaInfoKey(clientType, subject); + ClusterFuture future = clusterMap.get(key); + MetaInfoService.MetaInfoRequestParam requestParam = MetaInfoService.buildRequestParam(clientType, subject, group, appCode); + if (future == null) { + future = request(requestParam, false); + } else { + metaInfoService.tryAddRequest(requestParam); + } + return future.get(); + } + + @Override + public void refresh(ClientType clientType, String subject) { + refresh(clientType, subject, ""); + } + + @Override + public void refresh(ClientType clientType, String subject, String group) { + request(MetaInfoService.buildRequestParam(clientType, subject, group, appCode), true); + } + + @Override + public void setAppCode(String appCode) { + this.appCode = appCode; + } + + private ClusterFuture request(MetaInfoService.MetaInfoRequestParam requestParam, boolean refresh) { + String key = MapKeyBuilder.buildMetaInfoKey(requestParam.getClientType(), requestParam.getSubject()); + ClusterFuture newFuture = new ClusterFuture(); + ClusterFuture oldFuture = clusterMap.putIfAbsent(key, newFuture); + if (oldFuture != null) { + if (refresh && !oldFuture.inRequest.get()) { + oldFuture.inRequest.set(true); + metaInfoService.requestWrapper(requestParam); + } else { + metaInfoService.tryAddRequest(requestParam); + } + return oldFuture; + } + metaInfoService.requestWrapper(requestParam); + return newFuture; + } + + private static final class ClusterFuture { + private final CountDownLatch latch; + private final AtomicReference cluster; + private final AtomicBoolean inRequest; + + ClusterFuture() { + latch = new CountDownLatch(1); + cluster = new AtomicReference<>(null); + inRequest = new AtomicBoolean(true); + } + + ClusterFuture(BrokerClusterInfo cluster) { + latch = new CountDownLatch(0); + this.cluster = new AtomicReference<>(cluster); + inRequest = new AtomicBoolean(false); + } + + void set(BrokerClusterInfo cluster) { + this.cluster.set(cluster); + latch.countDown(); + inRequest.set(false); + } + + public BrokerClusterInfo get() { + while (true) { + try { + latch.await(); + break; + } catch (Exception e) { + LOGGER.warn("get broker cluster info be interrupted, and ignore"); + } + } + return cluster.get(); + } + } +} diff --git a/qmq-client/src/main/java/qunar/tc/qmq/broker/impl/PollBrokerLoadBalance.java b/qmq-client/src/main/java/qunar/tc/qmq/broker/impl/PollBrokerLoadBalance.java new file mode 100644 index 00000000..c5245942 --- /dev/null +++ b/qmq-client/src/main/java/qunar/tc/qmq/broker/impl/PollBrokerLoadBalance.java @@ -0,0 +1,79 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.broker.impl; + +import com.google.common.base.Supplier; +import com.google.common.base.Suppliers; +import qunar.tc.qmq.broker.BrokerClusterInfo; +import qunar.tc.qmq.broker.BrokerGroupInfo; +import qunar.tc.qmq.broker.BrokerLoadBalance; + +import java.util.List; +import java.util.concurrent.ThreadLocalRandom; + +/** + * @author yiqun.fan create on 17-8-18. + */ +public class PollBrokerLoadBalance implements BrokerLoadBalance { + + private static final Supplier SUPPLIER = Suppliers.memoize(new Supplier() { + @Override + public BrokerLoadBalance get() { + return new PollBrokerLoadBalance(); + } + }); + + public static BrokerLoadBalance getInstance() { + return SUPPLIER.get(); + } + + private PollBrokerLoadBalance() { + } + + @Override + public BrokerGroupInfo loadBalance(BrokerClusterInfo cluster, BrokerGroupInfo lastGroup) { + List groups = cluster.getGroups(); + if (lastGroup == null || lastGroup.getGroupIndex() < 0 || lastGroup.getGroupIndex() >= groups.size()) { + BrokerGroupInfo group; + for (int i = 0; i < groups.size(); i++) { + if ((group = selectRandom(groups)).isAvailable()) { + return group; + } + } + for (BrokerGroupInfo groupInfo : groups) { + if (groupInfo.isAvailable()) { + return groupInfo; + } + } + } else { + int index = lastGroup.getGroupIndex(); + for (int count = groups.size(); count > 0; count--) { + index = (index + 1) % groups.size(); + BrokerGroupInfo nextGroup = groups.get(index); + if (nextGroup.isAvailable()) { + return nextGroup; + } + } + } + return lastGroup; + } + + private BrokerGroupInfo selectRandom(List groups) { + int random = ThreadLocalRandom.current().nextInt(groups.size()); + return groups.get(random); + } +} diff --git a/qmq-client/src/main/java/qunar/tc/qmq/common/AtomicConfig.java b/qmq-client/src/main/java/qunar/tc/qmq/common/AtomicConfig.java new file mode 100644 index 00000000..e3d60fdd --- /dev/null +++ b/qmq-client/src/main/java/qunar/tc/qmq/common/AtomicConfig.java @@ -0,0 +1,80 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.common; + +import com.google.common.base.Optional; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.atomic.AtomicReference; + +/** + * @author yiqun.fan create on 17-8-18. + */ +public abstract class AtomicConfig { + private static final Logger LOGGER = LoggerFactory.getLogger(AtomicConfig.class); + + private final ConcurrentMap> configMap = new ConcurrentHashMap<>(); + + public void update(final String configName, final Map config) { + // update or add config + for (Map.Entry entry : config.entrySet()) { + setString(entry.getKey(), entry.getValue()); + LOGGER.info("set pull config: {}. {}={}", configName, entry.getKey(), entry.getValue()); + } + + // no configed + for (String key : configMap.keySet()) { + if (!config.containsKey(key)) { + AtomicReference valueRef = configMap.get(key); + if (valueRef != null) { + T oldValue = valueRef.get(); + updateValueOnNoConfiged(key, valueRef); + T newValue = valueRef.get(); + LOGGER.info("update no pull config: {}. key={}, oldValue={}, newValue={}", configName, key, oldValue, newValue); + } + } + } + } + + private void setString(String key, String value) { + Optional v = parse(key, value); + if (v.isPresent()) { + set(key, v.get()); + } + } + + private void set(String key, T value) { + AtomicReference old = configMap.putIfAbsent(key, new AtomicReference(value)); + if (old != null) { + old.set(value); + } + } + + public AtomicReference get(String key, T defaultValue) { + AtomicReference tmp = new AtomicReference<>(defaultValue); + AtomicReference old = configMap.putIfAbsent(key, tmp); + return old != null ? old : tmp; + } + + protected abstract Optional parse(String key, String value); + + protected abstract void updateValueOnNoConfiged(String key, AtomicReference valueRef); +} diff --git a/qmq-client/src/main/java/qunar/tc/qmq/common/AtomicIntegerConfig.java b/qmq-client/src/main/java/qunar/tc/qmq/common/AtomicIntegerConfig.java new file mode 100644 index 00000000..51d197cc --- /dev/null +++ b/qmq-client/src/main/java/qunar/tc/qmq/common/AtomicIntegerConfig.java @@ -0,0 +1,65 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.common; + +import com.google.common.base.Optional; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.concurrent.atomic.AtomicReference; + +/** + * @author yiqun.fan create on 17-8-17. + */ +public class AtomicIntegerConfig extends AtomicConfig { + private static final Logger LOGGER = LoggerFactory.getLogger(AtomicIntegerConfig.class); + + private final int defaultValue; + private final int minValue; + private final int maxValue; + + public AtomicIntegerConfig(int defaultValue, int minValue, int maxValue) { + this.defaultValue = defaultValue; + this.minValue = Math.min(minValue, maxValue); + this.maxValue = Math.max(minValue, maxValue); + } + + public AtomicReference get(String key) { + return get(key, defaultValue); + } + + @Override + public Optional parse(String key, String value) { + int v = defaultValue; + try { + v = Integer.parseInt(value); + } catch (NumberFormatException e) { + LOGGER.warn("parse config fail: {}={}, use default: {}", key, value, v); + } + if (v < minValue || v > maxValue) { + LOGGER.warn("config value {} out of range: [{}, {}], use default: {}", + v, minValue, maxValue, defaultValue); + v = defaultValue; + } + return Optional.of(v); + } + + @Override + protected void updateValueOnNoConfiged(String key, AtomicReference valueRef) { + valueRef.set(defaultValue); + } +} diff --git a/qmq-client/src/main/java/qunar/tc/qmq/common/ClientIdProvider.java b/qmq-client/src/main/java/qunar/tc/qmq/common/ClientIdProvider.java new file mode 100644 index 00000000..e423ee65 --- /dev/null +++ b/qmq-client/src/main/java/qunar/tc/qmq/common/ClientIdProvider.java @@ -0,0 +1,25 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.common; + +/** + * Created by zhaohui.yu + * 4/2/18 + */ +public interface ClientIdProvider { + String get(); +} diff --git a/qmq-client/src/main/java/qunar/tc/qmq/common/ClientIdProviderFactory.java b/qmq-client/src/main/java/qunar/tc/qmq/common/ClientIdProviderFactory.java new file mode 100644 index 00000000..2961afb8 --- /dev/null +++ b/qmq-client/src/main/java/qunar/tc/qmq/common/ClientIdProviderFactory.java @@ -0,0 +1,27 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.common; + +/** + * Created by zhaohui.yu + * 4/2/18 + */ +public class ClientIdProviderFactory { + public static ClientIdProvider createDefault() { + return new DefaultClientIdProvider(); + } +} diff --git a/qmq-client/src/main/java/qunar/tc/qmq/common/DefaultClientIdProvider.java b/qmq-client/src/main/java/qunar/tc/qmq/common/DefaultClientIdProvider.java new file mode 100644 index 00000000..456ff0f7 --- /dev/null +++ b/qmq-client/src/main/java/qunar/tc/qmq/common/DefaultClientIdProvider.java @@ -0,0 +1,64 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.common; + +import com.google.common.base.Charsets; +import com.google.common.base.Strings; +import com.google.common.hash.Hashing; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import qunar.tc.qmq.utils.NetworkUtils; + +import java.util.UUID; + +/** + * Created by zhaohui.yu + * 4/2/18 + */ +class DefaultClientIdProvider implements ClientIdProvider { + private static final Logger LOG = LoggerFactory.getLogger(ClientIdProvider.class); + + @Override + public String get() { + return NetworkUtils.getLocalHostname() + "@@" + defaultUniqueId(); + } + + private String defaultUniqueId() { + final String location = getPackageLocation(); + if (Strings.isNullOrEmpty(location)) { + return UUID.randomUUID().toString(); + } + + try { + return Hashing.md5().hashString(location, Charsets.UTF_8).toString(); + } catch (Exception e) { + LOG.error("compute md5sum for jar package location failed.", e); + } + + return UUID.randomUUID().toString(); + } + + private String getPackageLocation() { + try { + return ClientIdProvider.class.getProtectionDomain().getCodeSource().getLocation().getPath(); + } catch (Exception e) { + LOG.warn("get jar package location failed.", e); + } + + return ""; + } +} diff --git a/qmq-client/src/main/java/qunar/tc/qmq/common/MapKeyBuilder.java b/qmq-client/src/main/java/qunar/tc/qmq/common/MapKeyBuilder.java new file mode 100644 index 00000000..65b45b7e --- /dev/null +++ b/qmq-client/src/main/java/qunar/tc/qmq/common/MapKeyBuilder.java @@ -0,0 +1,48 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.common; + +import com.google.common.base.Splitter; +import com.google.common.base.Strings; + +import java.util.ArrayList; +import java.util.List; + +/** + * @author yiqun.fan create on 17-8-18. + */ +public class MapKeyBuilder { + private static final String SEPARATOR = "$"; + + private static final Splitter SPLITTER = Splitter.on(SEPARATOR).trimResults().omitEmptyStrings(); + + public static String buildSubscribeKey(String subject, String group) { + return Strings.nullToEmpty(subject) + SEPARATOR + Strings.nullToEmpty(group); + } + + public static String buildSenderKey(String brokerGroupName, String subject, String group) { + return brokerGroupName + MapKeyBuilder.SEPARATOR + buildSubscribeKey(subject, group); + } + + public static String buildMetaInfoKey(ClientType clientType, String subject) { + return clientType.name() + MapKeyBuilder.SEPARATOR + subject; + } + + public static List splitKey(String key) { + return Strings.isNullOrEmpty(key) ? new ArrayList() : SPLITTER.splitToList(key); + } +} diff --git a/qmq-client/src/main/java/qunar/tc/qmq/common/StatusSource.java b/qmq-client/src/main/java/qunar/tc/qmq/common/StatusSource.java new file mode 100644 index 00000000..966754c0 --- /dev/null +++ b/qmq-client/src/main/java/qunar/tc/qmq/common/StatusSource.java @@ -0,0 +1,37 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.common; + +/** + * Created by zhaohui.yu + * 4/9/18 + */ +public enum StatusSource { + HEALTHCHECKER((byte) 1), + OPS((byte) 2), + CODE((byte) 4); + + private byte code; + + StatusSource(byte code) { + this.code = code; + } + + public int getCode() { + return code; + } +} diff --git a/qmq-client/src/main/java/qunar/tc/qmq/common/SwitchWaiter.java b/qmq-client/src/main/java/qunar/tc/qmq/common/SwitchWaiter.java new file mode 100644 index 00000000..ef01941f --- /dev/null +++ b/qmq-client/src/main/java/qunar/tc/qmq/common/SwitchWaiter.java @@ -0,0 +1,90 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.common; + +import java.util.concurrent.TimeUnit; +import java.util.concurrent.locks.Condition; +import java.util.concurrent.locks.Lock; +import java.util.concurrent.locks.ReentrantLock; + +import static qunar.tc.qmq.common.StatusSource.*; + +/** + * @author yiqun.fan create on 17-8-18. + */ +public class SwitchWaiter { + + private static final int WAIT_TIMEOUT = 60000; + + private final Lock lock = new ReentrantLock(); + private final Condition condition = lock.newCondition(); + + private byte state; + + public SwitchWaiter(boolean initValue) { + this.state = (byte) (initValue ? 1 : 0); + this.state |= OPS.getCode(); + this.state |= CODE.getCode(); + } + + public void on(StatusSource src) { + change(src, true); + } + + public void off(StatusSource src) { + change(src, false); + } + + private void change(StatusSource src, boolean state) { + lock.lock(); + try { + boolean ori = (this.state & src.getCode()) == src.getCode(); + if (ori == state) return; + + if (state) { + this.state |= src.getCode(); + } else { + this.state &= ((~src.getCode()) & 7); + } + + condition.signalAll(); + } finally { + lock.unlock(); + } + } + + public boolean waitOn() { + lock.lock(); + try { + while (!isOnline()) { + try { + condition.await(WAIT_TIMEOUT, TimeUnit.MILLISECONDS); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + return false; + } + } + } finally { + lock.unlock(); + } + return true; + } + + private boolean isOnline() { + return this.state == 7; + } +} diff --git a/qmq-client/src/main/java/qunar/tc/qmq/common/TimerUtil.java b/qmq-client/src/main/java/qunar/tc/qmq/common/TimerUtil.java new file mode 100644 index 00000000..a799cf81 --- /dev/null +++ b/qmq-client/src/main/java/qunar/tc/qmq/common/TimerUtil.java @@ -0,0 +1,35 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.common; + +import io.netty.util.HashedWheelTimer; +import io.netty.util.Timeout; +import io.netty.util.TimerTask; +import qunar.tc.qmq.concurrent.NamedThreadFactory; + +import java.util.concurrent.TimeUnit; + +/** + * @author yiqun.fan create on 17-11-3. + */ +public class TimerUtil { + private static final HashedWheelTimer TIMER = new HashedWheelTimer(new NamedThreadFactory("qmq-timer")); + + public static Timeout newTimeout(TimerTask task, long delay, TimeUnit unit) { + return TIMER.newTimeout(task, delay, unit); + } +} diff --git a/qmq-client/src/main/java/qunar/tc/qmq/config/NettyClientConfigManager.java b/qmq-client/src/main/java/qunar/tc/qmq/config/NettyClientConfigManager.java new file mode 100644 index 00000000..e573eb86 --- /dev/null +++ b/qmq-client/src/main/java/qunar/tc/qmq/config/NettyClientConfigManager.java @@ -0,0 +1,39 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.config; + +import qunar.tc.qmq.netty.NettyClientConfig; + +/** + * @author yiqun.fan create on 17-7-4. + */ +public class NettyClientConfigManager { + private static final NettyClientConfigManager config = new NettyClientConfigManager(); + + public static NettyClientConfigManager get() { + return config; + } + + private volatile NettyClientConfig clientConfig = new NettyClientConfig(); + + private NettyClientConfigManager() { + } + + public NettyClientConfig getDefaultClientConfig() { + return clientConfig; + } +} diff --git a/qmq-client/src/main/java/qunar/tc/qmq/config/PullSubjectsConfig.java b/qmq-client/src/main/java/qunar/tc/qmq/config/PullSubjectsConfig.java new file mode 100644 index 00000000..a78793e9 --- /dev/null +++ b/qmq-client/src/main/java/qunar/tc/qmq/config/PullSubjectsConfig.java @@ -0,0 +1,158 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.config; + +import com.google.common.base.Strings; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import qunar.tc.qmq.common.AtomicConfig; +import qunar.tc.qmq.common.AtomicIntegerConfig; +import qunar.tc.qmq.configuration.DynamicConfig; +import qunar.tc.qmq.configuration.DynamicConfigLoader; +import qunar.tc.qmq.configuration.Listener; + +import java.util.HashMap; +import java.util.Map; +import java.util.TreeMap; +import java.util.concurrent.atomic.AtomicReference; + +/** + * @author yiqun.fan create on 17-8-17. + */ +public class PullSubjectsConfig { + private static final Logger LOGGER = LoggerFactory.getLogger(PullSubjectsConfig.class); + + private static final String PULL_SUBJECTS_CONFIG = "pull_subject_config.properties"; + + private static final PullSubjectsConfig config = new PullSubjectsConfig(); + private final Map configMap; + private final AtomicIntegerConfig pullBatchSizeConfig; + private final AtomicIntegerConfig pullTimeoutConfig; + private final AtomicIntegerConfig pullRequestTimeoutConfig; + private final AtomicIntegerConfig ackNosendLimit; + private final AtomicIntegerConfig maxRetryNum; + private final AtomicIntegerConfig refreshQueueCountIntervalConfig; + + private PullSubjectsConfig() { + pullBatchSizeConfig = new AtomicIntegerConfig(50, 1, 10000); + pullTimeoutConfig = new AtomicIntegerConfig(1000, 1000, Integer.MAX_VALUE); + pullRequestTimeoutConfig = new AtomicIntegerConfig(8000, 5000, Integer.MAX_VALUE); + ackNosendLimit = new AtomicIntegerConfig(100, Integer.MIN_VALUE, Integer.MAX_VALUE); + maxRetryNum = new AtomicIntegerConfig(32, 0, Integer.MAX_VALUE); + refreshQueueCountIntervalConfig = new AtomicIntegerConfig(5000, 1000, 600000); + + configMap = new HashMap<>(); + configMap.put(ConfigType.PULL_BATCHSIZE, pullBatchSizeConfig); + configMap.put(ConfigType.PULL_TIMEOUT, pullTimeoutConfig); + configMap.put(ConfigType.PULL_REQUEST_TIMEOUT, pullRequestTimeoutConfig); + configMap.put(ConfigType.ACK_NOSEND_LIMIT, ackNosendLimit); + configMap.put(ConfigType.MAX_RETRY_NUM, maxRetryNum); + configMap.put(ConfigType.QUEUE_COUNT_INTERVAL, refreshQueueCountIntervalConfig); + loadConfig(); + } + + public static PullSubjectsConfig get() { + return config; + } + + private void loadConfig() { + final DynamicConfig config = DynamicConfigLoader.load(PULL_SUBJECTS_CONFIG, false); + config.addListener(new Listener() { + @Override + public void onLoad(final DynamicConfig config) { + final Map originConf = config.asMap(); + + Map> subjectConfigMap = new TreeMap<>(); + for (ConfigType configType : ConfigType.values()) { + subjectConfigMap.put(configType, new TreeMap()); + } + + for (Map.Entry e : originConf.entrySet()) { + if (Strings.isNullOrEmpty(e.getKey()) || Strings.isNullOrEmpty(e.getValue())) { + continue; + } + String key = e.getKey(); + for (ConfigType configType : ConfigType.values()) { + if (key.endsWith(configType.suffix)) { + String subject = key.substring(0, key.length() - configType.suffix.length()); + if (Strings.isNullOrEmpty(subject)) { + LOGGER.warn("can't parse subject, please check config. {}={}", key, e.getValue()); + break; + } + Map subjectConfig = subjectConfigMap.get(configType); + if (subjectConfig == null) { + subjectConfig = new TreeMap<>(); + subjectConfigMap.put(configType, subjectConfig); + } + subjectConfig.put(subject, e.getValue()); + break; + } + } + } + + for (Map.Entry> e : subjectConfigMap.entrySet()) { + AtomicConfig atomicConfig = configMap.get(e.getKey()); + if (atomicConfig == null) { + continue; + } + atomicConfig.update(e.getKey().name(), e.getValue()); + } + + } + }); + } + + public AtomicReference getPullBatchSize(String subject) { + return pullBatchSizeConfig.get(subject); + } + + public AtomicReference getPullTimeout(String subject) { + return pullTimeoutConfig.get(subject); + } + + public AtomicReference getPullRequestTimeout(String subject) { + return pullRequestTimeoutConfig.get(subject); + } + + public AtomicReference getAckNosendLimit(String subject) { + return ackNosendLimit.get(subject); + } + + public AtomicReference getMaxRetryNum(String subject) { + return maxRetryNum.get(subject); + } + + public AtomicReference getRefreshQueueCountInterval(String subject) { + return refreshQueueCountIntervalConfig.get(subject); + } + + private enum ConfigType { + PULL_BATCHSIZE("_pullBatchSize"), + PULL_TIMEOUT("_pullTimeout"), + PULL_REQUEST_TIMEOUT("_pullRequestTimeout"), + ACK_TIMEOUT("_ackTimeout"), + ACK_NOSEND_LIMIT("_ackNosendLimit"), + MAX_RETRY_NUM("_maxRetryNum"), + QUEUE_COUNT_INTERVAL("_refreshQCInterval"); + + private final String suffix; + + ConfigType(String suffix) { + this.suffix = suffix; + } + } +} diff --git a/qmq-client/src/main/java/qunar/tc/qmq/consumer/BaseMessageHandler.java b/qmq-client/src/main/java/qunar/tc/qmq/consumer/BaseMessageHandler.java new file mode 100644 index 00000000..dc67616d --- /dev/null +++ b/qmq-client/src/main/java/qunar/tc/qmq/consumer/BaseMessageHandler.java @@ -0,0 +1,169 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.consumer; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import qunar.tc.qmq.*; +import qunar.tc.qmq.base.BaseMessage; +import qunar.tc.qmq.consumer.handler.IdempotentCheckerFilter; +import qunar.tc.qmq.consumer.handler.QTraceFilter; +import qunar.tc.qmq.tracing.TraceUtil; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.Executor; +import java.util.concurrent.RejectedExecutionException; + +/** + * @author yiqun.fan create on 17-8-18. + */ +public class BaseMessageHandler { + private static final Logger LOGGER = LoggerFactory.getLogger(BaseMessageHandler.class); + + protected final Executor executor; + protected final MessageListener listener; + private final List filters; + private final Filter qtraceFilter; + + public BaseMessageHandler(Executor executor, MessageListener listener) { + this.executor = executor; + this.listener = listener; + this.filters = new ArrayList<>(); + buildFilterChain(listener); + this.qtraceFilter = new QTraceFilter(); + } + + private void buildFilterChain(MessageListener listener) { + if (listener instanceof FilterAttachable) { + this.filters.addAll(FilterAttachable.class.cast(listener).filters()); + } + if (listener instanceof IdempotentAttachable) { + this.filters.add(new IdempotentCheckerFilter(IdempotentAttachable.class.cast(listener).getIdempotentChecker())); + } + } + + boolean triggerBeforeOnMessage(ConsumeMessage message, Map filterContext) { + for (int i = 0; i < filters.size(); ++i) { + message.processedFilterIndex(i); + if (!filters.get(i).preOnMessage(message, filterContext)) { + return false; + } + } + return true; + } + + protected void applyPostOnMessage(ConsumeMessage message, Throwable ex, Map filterContext) { + int processedFilterIndex = message.processedFilterIndex(); + for (int i = processedFilterIndex; i >= 0; --i) { + try { + filters.get(i).postOnMessage(message, ex, filterContext); + } catch (Throwable e) { + LOGGER.error("post filter failed", e); + } + } + } + + protected void ack(BaseMessage message, long elapsed, Throwable exception, Map attachment) { + + } + + public static void printError(BaseMessage message, Throwable e) { + if (e == null) return; + if (e instanceof NeedRetryException) return; + LOGGER.error("message process error. subject={}, msgId={}, times={}, maxRetryNum={}", + message.getSubject(), message.getMessageId(), message.times(), message.getMaxRetryNum(), e); + } + + public static class HandleTask implements Runnable { + protected final ConsumeMessage message; + private final BaseMessageHandler handler; + private volatile int localRetries = 0; // 本地重试次数 + protected volatile boolean handleFail = false; + + public HandleTask(ConsumeMessage message, BaseMessageHandler handler) { + this.message = message; + this.handler = handler; + } + + @Override + public void run() { + message.setProcessThread(Thread.currentThread()); + final long start = System.currentTimeMillis(); + final Map filterContext = new HashMap<>(); + message.localRetries(localRetries); + message.filterContext(filterContext); + + Throwable exception = null; + boolean reQueued = false; + try { + handler.qtraceFilter.preOnMessage(message, filterContext); + if (!handler.triggerBeforeOnMessage(message, filterContext)) return; + handler.listener.onMessage(message); + } catch (NeedRetryException e) { + exception = e; + try { + reQueued = localRetry(e); + } catch (Throwable ex) { + exception = ex; + } + } catch (Throwable e) { + exception = e; + } finally { + triggerAfterCompletion(reQueued, start, exception, filterContext); + handler.qtraceFilter.postOnMessage(message, exception, filterContext); + } + } + + private boolean localRetry(NeedRetryException e) { + boolean reQueued = false; + if (isRetryImmediately(e)) { + TraceUtil.recordEvent("local_retry"); + try { + ++localRetries; + handler.executor.execute(this); + reQueued = true; + } catch (RejectedExecutionException re) { + message.localRetries(localRetries); + try { + handler.listener.onMessage(message); + } catch (NeedRetryException ne) { + localRetry(ne); + } + } + } + return reQueued; + } + + private boolean isRetryImmediately(NeedRetryException e) { + long next = e.getNext(); + return next - System.currentTimeMillis() <= 50; + } + + private void triggerAfterCompletion(boolean reQueued, long start, Throwable exception, Map filterContext) { + handleFail = exception != null; + if (message.isAutoAck() || exception != null) { + handler.applyPostOnMessage(message, exception, filterContext); + + if (reQueued) return; + handler.ack(message, System.currentTimeMillis() - start, exception, null); + } + } + } +} diff --git a/qmq-client/src/main/java/qunar/tc/qmq/consumer/ConsumeMessage.java b/qmq-client/src/main/java/qunar/tc/qmq/consumer/ConsumeMessage.java new file mode 100644 index 00000000..02a2dd83 --- /dev/null +++ b/qmq-client/src/main/java/qunar/tc/qmq/consumer/ConsumeMessage.java @@ -0,0 +1,84 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.consumer; + +import qunar.tc.qmq.Message; +import qunar.tc.qmq.base.BaseMessage; + +import java.util.Map; + +/** + * @author yiqun.fan create on 17-8-19. + */ +public class ConsumeMessage extends BaseMessage { + private transient volatile Thread processThread; + private volatile boolean autoAck = true; + private volatile transient int localRetries; + private transient volatile Map filterContext; + private transient volatile int processedFilterIndex = -1; + + protected ConsumeMessage(BaseMessage message) { + super(message); + this.processThread = Thread.currentThread(); + } + + boolean isAutoAck() { + return this.autoAck; + } + + void setProcessThread(Thread processThread) { + this.processThread = processThread; + } + + @Override + public void autoAck(boolean auto) { + if (processThread != Thread.currentThread()) { + throw new RuntimeException("如需要使用显式ack,请在MessageListener的onMessage入口处调用autoAck设置,也不能在其他线程里调用"); + } + this.autoAck = auto; + } + + public Map filterContext() { + return filterContext; + } + + void filterContext(Map filterContext) { + this.filterContext = filterContext; + } + + int processedFilterIndex() { + return processedFilterIndex; + } + + void processedFilterIndex(int processedFilterIndex) { + this.processedFilterIndex = processedFilterIndex; + } + + @Override + public int localRetries() { + return localRetries; + } + + void localRetries(int localRetries) { + this.localRetries = localRetries; + } + + @Override + public Message addTag(String tag) { + throw new UnsupportedOperationException("use addTag in producer only"); + } +} diff --git a/qmq-client/src/main/java/qunar/tc/qmq/consumer/MessageConsumerProvider.java b/qmq-client/src/main/java/qunar/tc/qmq/consumer/MessageConsumerProvider.java new file mode 100644 index 00000000..a40728d6 --- /dev/null +++ b/qmq-client/src/main/java/qunar/tc/qmq/consumer/MessageConsumerProvider.java @@ -0,0 +1,148 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ +package qunar.tc.qmq.consumer; + +import com.google.common.base.Preconditions; +import com.google.common.base.Strings; +import qunar.tc.qmq.*; +import qunar.tc.qmq.common.ClientIdProvider; +import qunar.tc.qmq.common.ClientIdProviderFactory; +import qunar.tc.qmq.config.NettyClientConfigManager; +import qunar.tc.qmq.consumer.handler.MessageDistributor; +import qunar.tc.qmq.consumer.pull.PullConsumerFactory; +import qunar.tc.qmq.consumer.pull.PullRegister; +import qunar.tc.qmq.netty.client.NettyClient; + +import javax.annotation.PostConstruct; +import javax.annotation.PreDestroy; +import java.util.concurrent.Executor; + +/** + * @author miao.yang susing@gmail.com + * @date 2012-12-28 + */ +public class MessageConsumerProvider implements MessageConsumer { + + private static final int MAX_CONSUMER_GROUP_LEN = 50; + private static final int MAX_PREFIX_LEN = 100; + + private MessageDistributor distributor; + + private final PullConsumerFactory pullConsumerFactory; + + private volatile boolean inited = false; + + private ClientIdProvider clientIdProvider; + + private final PullRegister pullRegister; + private String appCode; + private String metaServer; + private int destroyWaitInSeconds; + + public MessageConsumerProvider() { + this.clientIdProvider = ClientIdProviderFactory.createDefault(); + this.pullRegister = new PullRegister(); + this.pullConsumerFactory = new PullConsumerFactory(this.pullRegister); + } + + @PostConstruct + public void init() { + Preconditions.checkNotNull(appCode, "appCode是应用的唯一标识"); + Preconditions.checkNotNull(metaServer, "metaServer是meta server的地址"); + + if (inited) return; + + synchronized (this) { + if (inited) return; + + NettyClient.getClient().start(NettyClientConfigManager.get().getDefaultClientConfig()); + + String clientId = this.clientIdProvider.get(); + this.pullRegister.setDestroyWaitInSeconds(destroyWaitInSeconds); + this.pullRegister.setMetaServer(metaServer); + this.pullRegister.setClientId(clientId); + this.pullRegister.init(); + + distributor = new MessageDistributor(pullRegister); + distributor.setClientId(clientId); + + pullRegister.setAutoOnline(true); + inited = true; + } + } + + @Override + public ListenerHolder addListener(String subject, String consumerGroup, MessageListener listener, Executor executor) { + return addListener(subject, consumerGroup, listener, executor, SubscribeParam.DEFAULT); + } + + @Override + public ListenerHolder addListener(String subject, String consumerGroup, MessageListener listener, Executor executor, SubscribeParam subscribeParam) { + init(); + Preconditions.checkArgument(subject != null && subject.length() <= MAX_PREFIX_LEN, "subjectPrefix长度不允许超过" + MAX_PREFIX_LEN + "个字符"); + Preconditions.checkArgument(consumerGroup == null || consumerGroup.length() <= MAX_CONSUMER_GROUP_LEN, "consumerGroup长度不允许超过" + MAX_CONSUMER_GROUP_LEN + "个字符"); + + Preconditions.checkArgument(!subject.contains("${"), "请确保subject已经正确解析: " + subject); + Preconditions.checkArgument(consumerGroup == null || !consumerGroup.contains("${"), "请确保consumerGroup已经正确解析: " + consumerGroup); + + if (Strings.isNullOrEmpty(consumerGroup)) { + subscribeParam.setBroadcast(true); + } + + if (subscribeParam.isBroadcast()) { + consumerGroup = clientIdProvider.get(); + } + + Preconditions.checkNotNull(executor, "消费逻辑将在该线程池里执行"); + Preconditions.checkNotNull(subscribeParam, "订阅时候的参数需要指定,如果使用默认参数的话请使用无此参数的重载"); + + return distributor.addListener(subject, consumerGroup, listener, executor, subscribeParam); + } + + @Override + public PullConsumer getOrCreatePullConsumer(String subject, String group, boolean isBroadcast) { + init(); + + Preconditions.checkArgument(!Strings.isNullOrEmpty(subject), "subject不能是nullOrEmpty"); + if (!isBroadcast) { + Preconditions.checkArgument(!Strings.isNullOrEmpty(group), "非广播订阅时,group不能是nullOrEmpty"); + } else { + group = clientIdProvider.get(); + } + return pullConsumerFactory.getOrCreateDefault(subject, group, isBroadcast); + } + + public void setClientIdProvider(ClientIdProvider clientIdProvider) { + this.clientIdProvider = clientIdProvider; + } + + public void setAppCode(String appCode) { + this.appCode = appCode; + } + + public void setMetaServer(String metaServer) { + this.metaServer = metaServer; + } + + public void setDestroyWaitInSeconds(int destroyWaitInSeconds) { + this.destroyWaitInSeconds = destroyWaitInSeconds; + } + + @PreDestroy + public void destroy() { + pullRegister.destroy(); + } +} diff --git a/qmq-client/src/main/java/qunar/tc/qmq/consumer/annotation/ConsumerAnnotationScanner.java b/qmq-client/src/main/java/qunar/tc/qmq/consumer/annotation/ConsumerAnnotationScanner.java new file mode 100644 index 00000000..a345360f --- /dev/null +++ b/qmq-client/src/main/java/qunar/tc/qmq/consumer/annotation/ConsumerAnnotationScanner.java @@ -0,0 +1,204 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.consumer.annotation; + +import com.google.common.base.Strings; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.BeansException; +import org.springframework.beans.factory.BeanFactory; +import org.springframework.beans.factory.BeanFactoryAware; +import org.springframework.beans.factory.config.BeanPostProcessor; +import org.springframework.beans.factory.support.AbstractBeanFactory; +import org.springframework.context.ApplicationContext; +import org.springframework.context.ApplicationContextAware; +import org.springframework.core.annotation.AnnotationUtils; +import qunar.tc.qmq.*; +import qunar.tc.qmq.consumer.MessageConsumerProvider; + +import java.lang.reflect.Method; +import java.lang.reflect.Modifier; +import java.util.*; +import java.util.concurrent.Executor; + +import static qunar.tc.qmq.consumer.annotation.QmqClientBeanDefinitionParser.DEFAULT_ID; + +class ConsumerAnnotationScanner implements BeanPostProcessor, ApplicationContextAware, BeanFactoryAware { + private static final Logger logger = LoggerFactory.getLogger(ConsumerAnnotationScanner.class); + + private static final Set registeredMethods = new HashSet(); + + private ApplicationContext context; + + private AbstractBeanFactory beanFactory; + + @Override + public Object postProcessBeforeInitialization(Object bean, String beanName) throws BeansException { + return bean; + } + + @Override + public Object postProcessAfterInitialization(Object bean, String beanName) throws BeansException { + parseMethods(bean, bean.getClass().getDeclaredMethods()); + return bean; + } + + private void parseMethods(final Object bean, Method[] methods) { + String beanName = bean.getClass().getCanonicalName(); + + for (final Method method : methods) { + if (registeredMethods.contains(method)) continue; + + QmqConsumer annotation = AnnotationUtils.findAnnotation(method, QmqConsumer.class); + if (annotation == null) continue; + + if (!Modifier.isPublic(method.getModifiers())) { + throw new RuntimeException("标记QmqConsumer的方法必须是public的"); + } + + String methodName = method.getName(); + Class[] args = method.getParameterTypes(); + String message = String.format("如果想配置成为message listener,方法必须有且只有一个参数,类型必须为qunar.tc.qmq.Message类型: %s method:%s", beanName, methodName); + if (args.length != 1) { + logger.error(message); + throw new RuntimeException(message); + } + if (args[0] != Message.class) { + logger.error(message); + throw new RuntimeException(message); + } + + String subject = resolve(annotation.subject()); + + if (Strings.isNullOrEmpty(subject)) { + String err = String.format("使用@QmqConsumer,必须提供prefix, class:%s method:%s", beanName, methodName); + logger.error(err); + throw new RuntimeException(err); + } + registeredMethods.add(method); + + String consumerGroup = annotation.isBroadcast() ? "" : resolve(annotation.consumerGroup()); + ListenerHolder listenerHolder = new ListenerHolder(context, beanFactory, bean, method, subject, consumerGroup, annotation.executor(), buildSubscribeParam(annotation), annotation.idempotentChecker(), annotation.filters()); + listenerHolder.registe(); + } + } + + private SubscribeParam buildSubscribeParam(QmqConsumer annotation) { + return new SubscribeParam.SubscribeParamBuilder() + .setConsumeMostOnce(annotation.consumeMostOnce()) + .setTags(new HashSet<>(Arrays.asList(annotation.tags()))) + .setTagType(annotation.tagType()) + .create(); + } + + private String resolve(String value) { + if (Strings.isNullOrEmpty(value)) return value; + if (beanFactory == null) return value; + return beanFactory.resolveEmbeddedValue(value); + } + + @Override + public void setApplicationContext(ApplicationContext applicationContext) throws BeansException { + this.context = applicationContext; + } + + @Override + public void setBeanFactory(BeanFactory beanFactory) throws BeansException { + if (beanFactory instanceof AbstractBeanFactory) { + this.beanFactory = (AbstractBeanFactory) beanFactory; + } + } + + private static class ListenerHolder { + private final ApplicationContext context; + private final MessageListener listener; + private final AbstractBeanFactory beanFactory; + private final String subject; + private final String group; + private final String executorName; + private final SubscribeParam subscribeParam; + + private ListenerHolder(ApplicationContext context, + AbstractBeanFactory beanFactory, + Object bean, + Method method, + String subject, + String group, + String executor, + SubscribeParam subscribeParam, + String idempotentChecker, + String[] filters) { + this.context = context; + this.beanFactory = beanFactory; + this.subject = subject; + this.group = group; + this.executorName = executor; + this.subscribeParam = subscribeParam; + IdempotentChecker idempotentCheckerBean = null; + if (idempotentChecker != null && idempotentChecker.length() > 0) { + idempotentCheckerBean = context.getBean(idempotentChecker, IdempotentChecker.class); + } + + List filterBeans = new ArrayList<>(); + if (filters != null && filters.length > 0) { + for (int i = 0; i < filters.length; ++i) { + filterBeans.add(context.getBean(filters[i], Filter.class)); + } + } + this.listener = new GeneratedListener(bean, method, idempotentCheckerBean, filterBeans); + } + + public void registe() { + MessageConsumerProvider consumer = resolveConsumer(); + Executor executor = resolveExecutor(executorName); + consumer.addListener(subject, group, listener, executor, subscribeParam); + } + + private MessageConsumerProvider resolveConsumer() { + MessageConsumerProvider result = null; + + if (beanFactory != null) { + result = beanFactory.getBean(DEFAULT_ID, MessageConsumerProvider.class); + } + if (result != null) return result; + + if (context != null) { + result = context.getBean(DEFAULT_ID, MessageConsumerProvider.class); + } + if (result != null) return result; + + throw new RuntimeException("没有正确的配置qmq,如果使用Springboot请确保升级到了最新版本"); + } + + private Executor resolveExecutor(String executorName) { + Executor executor = null; + + if (beanFactory != null) { + executor = beanFactory.getBean(executorName, Executor.class); + } + + if (executor != null) return executor; + + if (context != null) { + executor = context.getBean(executorName, Executor.class); + } + if (executor != null) return executor; + + throw new RuntimeException("处理消息的线程池必须配置"); + } + } +} diff --git a/qmq-client/src/main/java/qunar/tc/qmq/consumer/annotation/EnableQmq.java b/qmq-client/src/main/java/qunar/tc/qmq/consumer/annotation/EnableQmq.java new file mode 100644 index 00000000..bb76a4db --- /dev/null +++ b/qmq-client/src/main/java/qunar/tc/qmq/consumer/annotation/EnableQmq.java @@ -0,0 +1,36 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.consumer.annotation; + +import org.springframework.context.annotation.Import; + +import java.lang.annotation.*; + +/** + * Created by zhaohui.yu + * 10/11/17 + */ +@Retention(RetentionPolicy.RUNTIME) +@Target(ElementType.TYPE) +@Documented +@Import(QmqConsumerRegister.class) +public @interface EnableQmq { + + String appCode(); + + String metaServer(); +} diff --git a/qmq-client/src/main/java/qunar/tc/qmq/consumer/annotation/GeneratedListener.java b/qmq-client/src/main/java/qunar/tc/qmq/consumer/annotation/GeneratedListener.java new file mode 100644 index 00000000..9acbb843 --- /dev/null +++ b/qmq-client/src/main/java/qunar/tc/qmq/consumer/annotation/GeneratedListener.java @@ -0,0 +1,60 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.consumer.annotation; + +import qunar.tc.qmq.*; + +import java.lang.reflect.Method; +import java.util.List; + +/** + * User: zhaohuiyu + * Date: 9/16/14 + * Time: 3:16 PM + */ +class GeneratedListener implements MessageListener, FilterAttachable, IdempotentAttachable { + private final Object bean; + private final Method method; + private final IdempotentChecker idempotentChecker; + private final List filters; + + GeneratedListener(Object bean, Method method, IdempotentChecker idempotentChecker, List filters) { + this.bean = bean; + this.method = method; + this.idempotentChecker = idempotentChecker; + this.filters = filters; + } + + @Override + public void onMessage(Message msg) { + try { + this.method.invoke(bean, msg); + } catch (Exception e) { + throw new RuntimeException("processor message error", e); + } + } + + @Override + public List filters() { + return filters; + } + + @Override + public IdempotentChecker getIdempotentChecker() { + return idempotentChecker; + } +} diff --git a/qmq-client/src/main/java/qunar/tc/qmq/consumer/annotation/QmqClientBeanDefinitionParser.java b/qmq-client/src/main/java/qunar/tc/qmq/consumer/annotation/QmqClientBeanDefinitionParser.java new file mode 100644 index 00000000..b718740f --- /dev/null +++ b/qmq-client/src/main/java/qunar/tc/qmq/consumer/annotation/QmqClientBeanDefinitionParser.java @@ -0,0 +1,57 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.consumer.annotation; + +import org.springframework.beans.factory.support.AbstractBeanDefinition; +import org.springframework.beans.factory.support.BeanDefinitionBuilder; +import org.springframework.beans.factory.support.GenericBeanDefinition; +import org.springframework.beans.factory.xml.AbstractSingleBeanDefinitionParser; +import org.springframework.beans.factory.xml.ParserContext; +import org.w3c.dom.Element; +import qunar.tc.qmq.consumer.MessageConsumerProvider; + +class QmqClientBeanDefinitionParser extends AbstractSingleBeanDefinitionParser { + static final String QMQ_CLIENT_ANNOTATION = "QMQ_CLIENT_ANNOTATION"; + + static final String DEFAULT_ID = "QMQ_CONSUMER_ALL"; + + @Override + protected Class getBeanClass(Element element) { + return MessageConsumerProvider.class; + } + + @Override + protected void doParse(Element element, ParserContext parserContext, BeanDefinitionBuilder builder) { + String appCode = element.getAttribute("appCode"); + String metaServer = element.getAttribute("metaServer"); + + builder.addPropertyValue("appCode", appCode); + builder.addPropertyValue("metaServer", metaServer); + + if (!parserContext.getRegistry().containsBeanDefinition(QMQ_CLIENT_ANNOTATION)) { + GenericBeanDefinition scanner = new GenericBeanDefinition(); + scanner.setBeanClass(ConsumerAnnotationScanner.class); + parserContext.getRegistry().registerBeanDefinition(QMQ_CLIENT_ANNOTATION, scanner); + } + } + + @Override + protected String resolveId(Element element, AbstractBeanDefinition definition, ParserContext parserContext) { + return DEFAULT_ID; + } + +} diff --git a/qmq-client/src/main/java/qunar/tc/qmq/consumer/annotation/QmqClientNamespaceHandler.java b/qmq-client/src/main/java/qunar/tc/qmq/consumer/annotation/QmqClientNamespaceHandler.java new file mode 100644 index 00000000..9330a67f --- /dev/null +++ b/qmq-client/src/main/java/qunar/tc/qmq/consumer/annotation/QmqClientNamespaceHandler.java @@ -0,0 +1,31 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.consumer.annotation; + +import org.springframework.beans.factory.xml.NamespaceHandlerSupport; + +/** + * User: zhaohuiyu + * Date: 7/5/13 + * Time: 5:37 PM + */ +class QmqClientNamespaceHandler extends NamespaceHandlerSupport { + @Override + public void init() { + registerBeanDefinitionParser("consumer", new QmqClientBeanDefinitionParser()); + } +} diff --git a/qmq-client/src/main/java/qunar/tc/qmq/consumer/annotation/QmqConsumer.java b/qmq-client/src/main/java/qunar/tc/qmq/consumer/annotation/QmqConsumer.java new file mode 100644 index 00000000..bbc071c5 --- /dev/null +++ b/qmq-client/src/main/java/qunar/tc/qmq/consumer/annotation/QmqConsumer.java @@ -0,0 +1,85 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.consumer.annotation; + +import qunar.tc.qmq.TagType; + +import java.lang.annotation.*; + +/** + * User: zhaohuiyu + * Date: 7/5/13 + * Time: 7:28 PM + */ +@Documented +@Retention(RetentionPolicy.RUNTIME) +@Target({ElementType.METHOD}) +@Inherited +public @interface QmqConsumer { + /** + * (必填)订阅的主题, 支持${subject}形式注入 + */ + String subject(); + + /** + * (可选)consumerGroup,如果不填则为广播模式, 支持${group}形式注入 + */ + String consumerGroup() default ""; + + /** + * 注入ThreadPoolExecutor bean,消费消息逻辑在此线程池执行 + * + * @return + */ + String executor(); + + /** + * 是否是广播消息 + * 设成true时, 忽略consumerGroup的值 + */ + boolean isBroadcast() default false; + + /** + * 设成true时,每条消息(包括可靠)最多消费一次 + */ + boolean consumeMostOnce() default false; + + /** + * 设置过滤tags。格式是{tag1, tag2, ...} + * tags的长度为0时,过滤会失效,消费所有消息 + * 并且需要设置TagType,默认TagType是TagType.NO_TAG,tags会失效 + */ + String[] tags() default {}; + + /** + * 默认是TagType.NO_TAG + * 如果TagType设置成TagType.NO_TAG, 则tags会失效,消费所有消息。 + * TagType.OR 表示以上设置的tags与消息的tags有交集 + * TagType.AND 表示以上设置的tags是消息的tags的子集 + */ + TagType tagType() default TagType.NO_TAG; + + /** + * 幂等检查器beanName + */ + String idempotentChecker() default ""; + + /** + * 过滤器beanName,按顺序 + */ + String[] filters() default {}; +} diff --git a/qmq-client/src/main/java/qunar/tc/qmq/consumer/annotation/QmqConsumerRegister.java b/qmq-client/src/main/java/qunar/tc/qmq/consumer/annotation/QmqConsumerRegister.java new file mode 100644 index 00000000..3f52c390 --- /dev/null +++ b/qmq-client/src/main/java/qunar/tc/qmq/consumer/annotation/QmqConsumerRegister.java @@ -0,0 +1,63 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.consumer.annotation; + +import org.springframework.beans.MutablePropertyValues; +import org.springframework.beans.factory.support.BeanDefinitionRegistry; +import org.springframework.beans.factory.support.GenericBeanDefinition; +import org.springframework.context.annotation.ImportBeanDefinitionRegistrar; +import org.springframework.core.annotation.AnnotationAttributes; +import org.springframework.core.type.AnnotationMetadata; +import qunar.tc.qmq.consumer.MessageConsumerProvider; + +import static qunar.tc.qmq.consumer.annotation.QmqClientBeanDefinitionParser.DEFAULT_ID; +import static qunar.tc.qmq.consumer.annotation.QmqClientBeanDefinitionParser.QMQ_CLIENT_ANNOTATION; + +/** + * Created by zhaohui.yu + * 2/4/17 + */ +class QmqConsumerRegister implements ImportBeanDefinitionRegistrar { + + @Override + public void registerBeanDefinitions(AnnotationMetadata importingClassMetadata, BeanDefinitionRegistry registry) { + AnnotationAttributes attributes = AnnotationAttributes.fromMap(importingClassMetadata.getAnnotationAttributes(EnableQmq.class.getName())); + String appCode = attributes.getString("appCode"); + String metaServer = attributes.getString("metaServer"); + + if (!registry.containsBeanDefinition(DEFAULT_ID)) { + GenericBeanDefinition beanDefinition = new GenericBeanDefinition(); + beanDefinition.setBeanClass(MessageConsumerProvider.class); + beanDefinition.setLazyInit(true); + beanDefinition.setPropertyValues(propertyValues(appCode, metaServer)); + registry.registerBeanDefinition(DEFAULT_ID, beanDefinition); + } + + if (!registry.containsBeanDefinition(QMQ_CLIENT_ANNOTATION)) { + GenericBeanDefinition beanDefinition = new GenericBeanDefinition(); + beanDefinition.setBeanClass(ConsumerAnnotationScanner.class); + registry.registerBeanDefinition(QMQ_CLIENT_ANNOTATION, beanDefinition); + } + } + + private MutablePropertyValues propertyValues(String appCode, String metaServer) { + MutablePropertyValues propertyValues = new MutablePropertyValues(); + propertyValues.add("appCode", appCode); + propertyValues.add("metaServer", metaServer); + return propertyValues; + } +} diff --git a/qmq-client/src/main/java/qunar/tc/qmq/consumer/exception/CreatePullConsumerException.java b/qmq-client/src/main/java/qunar/tc/qmq/consumer/exception/CreatePullConsumerException.java new file mode 100644 index 00000000..c8b0fde0 --- /dev/null +++ b/qmq-client/src/main/java/qunar/tc/qmq/consumer/exception/CreatePullConsumerException.java @@ -0,0 +1,38 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.consumer.exception; + +/** + * @author yiqun.fan create on 17-9-20. + */ +public class CreatePullConsumerException extends RuntimeException { + public CreatePullConsumerException() { + super(); + } + + public CreatePullConsumerException(String message, String subject, String group) { + super(message + ". subject=" + subject + ", group=" + group); + } + + public CreatePullConsumerException(String message, Throwable cause) { + super(message, cause); + } + + public CreatePullConsumerException(Throwable cause) { + super(cause); + } +} diff --git a/qmq-client/src/main/java/qunar/tc/qmq/consumer/exception/DuplicateListenerException.java b/qmq-client/src/main/java/qunar/tc/qmq/consumer/exception/DuplicateListenerException.java new file mode 100644 index 00000000..435b3c0d --- /dev/null +++ b/qmq-client/src/main/java/qunar/tc/qmq/consumer/exception/DuplicateListenerException.java @@ -0,0 +1,36 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.consumer.exception; + +/** + * User: zhaohuiyu + * Date: 10/15/13 + * Time: 1:53 PM + */ +public class DuplicateListenerException extends RuntimeException { + private static final long serialVersionUID = 1377475808662809865L; + private String key; + + public DuplicateListenerException(String key) { + super(key); + this.key = key; + } + + public String getKey() { + return this.key; + } +} diff --git a/qmq-client/src/main/java/qunar/tc/qmq/consumer/handler/IdempotentCheckerFilter.java b/qmq-client/src/main/java/qunar/tc/qmq/consumer/handler/IdempotentCheckerFilter.java new file mode 100644 index 00000000..0eb4fd4c --- /dev/null +++ b/qmq-client/src/main/java/qunar/tc/qmq/consumer/handler/IdempotentCheckerFilter.java @@ -0,0 +1,55 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.consumer.handler; + +import qunar.tc.qmq.Filter; +import qunar.tc.qmq.IdempotentChecker; +import qunar.tc.qmq.Message; +import qunar.tc.qmq.tracing.TraceUtil; + +import java.util.Map; + +/** + * Created by zhaohui.yu + * 15/12/8 + */ +public class IdempotentCheckerFilter implements Filter { + + private final IdempotentChecker idempotentChecker; + + public IdempotentCheckerFilter(IdempotentChecker idempotentChecker) { + this.idempotentChecker = idempotentChecker; + } + + @Override + public boolean preOnMessage(Message message, Map filterContext) { + if (idempotentChecker == null) return true; + TraceUtil.recordEvent("start idempotent"); + boolean processed = idempotentChecker.isProcessed(message); + TraceUtil.recordEvent("end idempotent"); + if (processed) { + TraceUtil.setTag("idempotent", "processed"); + } + return !processed; + } + + @Override + public void postOnMessage(Message message, Throwable e, Map filterContext) { + if (idempotentChecker == null) return; + idempotentChecker.markProcessed(message, e); + } +} diff --git a/qmq-client/src/main/java/qunar/tc/qmq/consumer/handler/MessageDistributor.java b/qmq-client/src/main/java/qunar/tc/qmq/consumer/handler/MessageDistributor.java new file mode 100644 index 00000000..d2a980b0 --- /dev/null +++ b/qmq-client/src/main/java/qunar/tc/qmq/consumer/handler/MessageDistributor.java @@ -0,0 +1,67 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ +package qunar.tc.qmq.consumer.handler; + +import com.google.common.base.Preconditions; +import com.google.common.base.Strings; +import qunar.tc.qmq.ListenerHolder; +import qunar.tc.qmq.MessageListener; +import qunar.tc.qmq.SubscribeParam; +import qunar.tc.qmq.consumer.register.ConsumerRegister; +import qunar.tc.qmq.consumer.register.RegistParam; + +import java.util.concurrent.Executor; + +import static qunar.tc.qmq.common.StatusSource.CODE; + +/** + * @author miao.yang susing@gmail.com + * @date 2012-12-28 + */ +public class MessageDistributor { + private final ConsumerRegister register; + + private String clientId; + + public MessageDistributor(ConsumerRegister register) { + this.register = register; + } + + public ListenerHolder addListener(final String subjectPrefix, final String consumerGroup, MessageListener listener, Executor executor, SubscribeParam subscribeParam) { + Preconditions.checkArgument(!Strings.isNullOrEmpty(consumerGroup)); + + final RegistParam registParam = new RegistParam(executor, listener, subscribeParam, clientId); + registParam.setBroadcast(subscribeParam.isBroadcast()); + register.regist(subjectPrefix, consumerGroup, registParam); + return new ListenerHolder() { + + @Override + public void stopListen() { + register.unregist(subjectPrefix, consumerGroup); + } + + @Override + public void resumeListen() { + registParam.setActionSrc(CODE); + register.regist(subjectPrefix, consumerGroup, registParam); + } + }; + } + + public void setClientId(String clientId) { + this.clientId = clientId; + } +} diff --git a/qmq-client/src/main/java/qunar/tc/qmq/consumer/handler/QTraceFilter.java b/qmq-client/src/main/java/qunar/tc/qmq/consumer/handler/QTraceFilter.java new file mode 100644 index 00000000..74e7733d --- /dev/null +++ b/qmq-client/src/main/java/qunar/tc/qmq/consumer/handler/QTraceFilter.java @@ -0,0 +1,67 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.consumer.handler; + +import io.opentracing.Scope; +import io.opentracing.Span; +import io.opentracing.SpanContext; +import io.opentracing.Tracer; +import io.opentracing.util.GlobalTracer; +import qunar.tc.qmq.Filter; +import qunar.tc.qmq.Message; +import qunar.tc.qmq.tracing.TraceUtil; + +import java.util.Map; + +/** + * Created by zhaohui.yu + * 15/12/9 + */ +public class QTraceFilter implements Filter { + + private static final String TRACE_OBJECT = "qtracer"; + + private static final String TRACE_DESC = "Qmq.Consume.Process"; + + private final Tracer tracer; + + public QTraceFilter() { + tracer = GlobalTracer.get(); + } + + @Override + public boolean preOnMessage(Message message, Map filterContext) { + SpanContext context = TraceUtil.extract(message, tracer); + Scope scope = tracer.buildSpan(TRACE_DESC) + .withTag("subject", message.getSubject()) + .withTag("messageId", message.getMessageId()) + .withTag("localRetries", String.valueOf(message.localRetries())) + .withTag("times", String.valueOf(message.times())) + .asChildOf(context) + .startActive(true); + filterContext.put(TRACE_OBJECT, scope.span()); + return true; + } + + @Override + public void postOnMessage(Message message, Throwable e, Map filterContext) { + Object o = filterContext.get(TRACE_OBJECT); + if (!(o instanceof Span)) return; + Scope scope = tracer.scopeManager().activate((Span) o, true); + scope.close(); + } +} diff --git a/qmq-client/src/main/java/qunar/tc/qmq/consumer/idempotent/AbstractIdempotentChecker.java b/qmq-client/src/main/java/qunar/tc/qmq/consumer/idempotent/AbstractIdempotentChecker.java new file mode 100644 index 00000000..7308941b --- /dev/null +++ b/qmq-client/src/main/java/qunar/tc/qmq/consumer/idempotent/AbstractIdempotentChecker.java @@ -0,0 +1,142 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.consumer.idempotent; + +import com.google.common.base.Function; +import com.google.common.base.Preconditions; +import com.google.common.base.Strings; +import qunar.tc.qmq.IdempotentChecker; +import qunar.tc.qmq.Message; +import qunar.tc.qmq.base.BaseMessage; + +import java.util.Date; + +/** + * Created by zhaohui.yu + * 15/11/25 + */ +public abstract class AbstractIdempotentChecker implements IdempotentChecker { + + private final Function keyFunc; + + public AbstractIdempotentChecker(Function keyFunc) { + Preconditions.checkNotNull(keyFunc, "用于生成幂等key的函数不能为空"); + this.keyFunc = keyFunc; + } + + @Override + public final boolean isProcessed(Message message) { + try { + return doIsProcessed(message); + } catch (Exception e) { + if (isIgnoreOnFailed()) { + return false; + } + throw new RuntimeException(e); + } + } + + protected abstract boolean doIsProcessed(Message message) throws Exception; + + @Override + public final void markProcessed(Message message, Throwable e) { + if (e == null) { + markProcessed(message); + return; + } + + if (e instanceof Exception) { + //忽略某些异常 + if (idempotentFor != null) { + if (idempotentFor.isAssignableFrom(e.getClass())) { + markProcessed(message); + return; + } + } + + //只有指定的异常回滚 + if (retryFor != null) { + if (retryFor.isAssignableFrom(e.getClass())) { + markFailed(message); + return; + } + } + markFailed(message); + } else { + markProcessed(message); + } + } + + protected abstract void markFailed(Message message); + + protected abstract void markProcessed(Message message); + + private Class retryFor; + private Class idempotentFor; + private boolean ignoreOnFailed; + + public final void setRetryFor(Class e) { + this.retryFor = e; + } + + public final void setIdempotentFor(Class e) { + this.idempotentFor = e; + } + + private boolean needRetry(Exception e) { + if (retryFor == null) return true; + return (retryFor.isAssignableFrom(e.getClass())); + } + + public final boolean isIdempotent(Exception e) { + if (idempotentFor == null) return false; + return idempotentFor.isAssignableFrom(e.getClass()); + } + + /** + * 这个地方将subject, consumerGroup拼接上去太长了 + * 最好是能将这个长串映射为一个整型带过来 + * + * @param message + * @return + */ + protected String keyOf(Message message) { + String original = keyFunc.apply(message); + Preconditions.checkArgument(!Strings.isNullOrEmpty(original), "使用所提供的keyFunc无法提取幂等key"); + + return message.getSubject() + "%" + message.getStringProperty(BaseMessage.keys.qmq_consumerGroupName.name()) + "%" + original; + } + + + public final void setIgnoreOnFailed(boolean ignoreOnFailed) { + this.ignoreOnFailed = ignoreOnFailed; + } + + protected final boolean isIgnoreOnFailed() { + return this.ignoreOnFailed; + } + + public abstract void garbageCollect(Date before); + + public static Function DEFAULT_KEYFUNC = new Function() { + @Override + public String apply(Message input) { + return input.getMessageId(); + } + }; + +} diff --git a/qmq-client/src/main/java/qunar/tc/qmq/consumer/idempotent/JdbcIdempotentChecker.java b/qmq-client/src/main/java/qunar/tc/qmq/consumer/idempotent/JdbcIdempotentChecker.java new file mode 100644 index 00000000..6d634496 --- /dev/null +++ b/qmq-client/src/main/java/qunar/tc/qmq/consumer/idempotent/JdbcIdempotentChecker.java @@ -0,0 +1,77 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.consumer.idempotent; + +import com.google.common.base.Function; +import org.springframework.jdbc.core.JdbcTemplate; +import qunar.tc.qmq.Message; + +import javax.sql.DataSource; +import java.util.Date; + +/** + * Created by zhaohui.yu + * 15/11/25 + *

+ * 使用数据库作为幂等检查的存储 + */ +public class JdbcIdempotentChecker extends AbstractIdempotentChecker { + + private static final String INSERT_TEMP = "INSERT IGNORE INTO %s(k) VALUES(?)"; + private static final String DELETE_TEMP = "DELETE FROM %s WHERE k=?"; + private static final String GARBAGE_TEMP = "DELETE FROM %s WHERE update_at keyFunc) { + super(keyFunc); + this.jdbcTemplate = new JdbcTemplate(dataSource); + this.INSERT_SQL = String.format(INSERT_TEMP, tableName); + this.DELETE_SQL = String.format(DELETE_TEMP, tableName); + this.GARBAGE_SQL = String.format(GARBAGE_TEMP, tableName); + } + + @Override + protected boolean doIsProcessed(Message message) throws Exception { + int update = jdbcTemplate.update(INSERT_SQL, keyOf(message)); + if (update == 1) return false; + return true; + } + + @Override + protected void markFailed(Message message) { + jdbcTemplate.update(DELETE_SQL, keyOf(message)); + } + + @Override + protected void markProcessed(Message message) { + + } + + @Override + public void garbageCollect(Date before) { + this.jdbcTemplate.update(GARBAGE_SQL, before); + } +} diff --git a/qmq-client/src/main/java/qunar/tc/qmq/consumer/idempotent/TransactionalJdbcIdempotentChecker.java b/qmq-client/src/main/java/qunar/tc/qmq/consumer/idempotent/TransactionalJdbcIdempotentChecker.java new file mode 100644 index 00000000..09a78b21 --- /dev/null +++ b/qmq-client/src/main/java/qunar/tc/qmq/consumer/idempotent/TransactionalJdbcIdempotentChecker.java @@ -0,0 +1,72 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.consumer.idempotent; + +import com.google.common.base.Function; +import org.springframework.jdbc.datasource.DataSourceTransactionManager; +import org.springframework.transaction.TransactionStatus; +import org.springframework.transaction.interceptor.DefaultTransactionAttribute; +import qunar.tc.qmq.Message; + +import java.util.Date; + +/** + * Created by zhaohui.yu + * 15/11/30 + *

+ * 可以将幂等检查与业务操作放在同一个事务里(如果业务里只有数据库操作),这种方式是最推荐的 + * 可以做到消息仅消费一次 + */ +public class TransactionalJdbcIdempotentChecker extends AbstractIdempotentChecker { + private final DataSourceTransactionManager transactionManager; + private final AbstractIdempotentChecker idempotentChecker; + + private static final ThreadLocal currentStatus = new ThreadLocal<>(); + + public TransactionalJdbcIdempotentChecker(DataSourceTransactionManager transactionManager, String tableName) { + this(transactionManager, tableName, DEFAULT_KEYFUNC); + } + + public TransactionalJdbcIdempotentChecker(DataSourceTransactionManager transactionManager, String tableName, Function keyFunc) { + super(keyFunc); + this.transactionManager = transactionManager; + this.idempotentChecker = new JdbcIdempotentChecker(transactionManager.getDataSource(), tableName, keyFunc); + } + + @Override + protected boolean doIsProcessed(Message message) throws Exception { + currentStatus.set(this.transactionManager.getTransaction(new DefaultTransactionAttribute())); + return idempotentChecker.doIsProcessed(message); + } + + @Override + protected void markFailed(Message message) { + TransactionStatus status = currentStatus.get(); + this.transactionManager.rollback(status); + } + + @Override + protected void markProcessed(Message message) { + TransactionStatus status = currentStatus.get(); + this.transactionManager.commit(status); + } + + @Override + public void garbageCollect(Date before) { + this.idempotentChecker.garbageCollect(before); + } +} diff --git a/qmq-client/src/main/java/qunar/tc/qmq/consumer/pull/AbstractPullConsumer.java b/qmq-client/src/main/java/qunar/tc/qmq/consumer/pull/AbstractPullConsumer.java new file mode 100644 index 00000000..cca58a09 --- /dev/null +++ b/qmq-client/src/main/java/qunar/tc/qmq/consumer/pull/AbstractPullConsumer.java @@ -0,0 +1,126 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.consumer.pull; + +import com.google.common.util.concurrent.ListenableFuture; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import qunar.tc.qmq.Message; +import qunar.tc.qmq.PullConsumer; +import qunar.tc.qmq.broker.BrokerService; +import qunar.tc.qmq.common.StatusSource; +import qunar.tc.qmq.common.SwitchWaiter; + +import java.util.List; +import java.util.concurrent.Future; + +import static qunar.tc.qmq.common.StatusSource.CODE; + +/** + * @author yiqun.fan create on 17-10-19. + */ +abstract class AbstractPullConsumer implements PullConsumer { + private static final Logger LOGGER = LoggerFactory.getLogger(AbstractPullConsumer.class); + + private static final long MIN_PULL_TIMEOUT_MILLIS = 1000; // 最短拉取超时时间是1秒 + private static long MAX_PULL_TIMEOUT_MILLIS = Long.MAX_VALUE / 2; // 最长拉取超时时间 + + final SwitchWaiter onlineSwitcher = new SwitchWaiter(true); + + private final ConsumeParam consumeParam; + private final ConsumeParam retryConsumeParam; + final PlainPullEntry pullEntry; + final PlainPullEntry retryPullEntry; + + AbstractPullConsumer(String subject, String group, boolean isBroadcast, String clientId, PullService pullService, AckService ackService, BrokerService brokerService) { + this.consumeParam = new ConsumeParam(subject, group, isBroadcast, false, clientId); + this.retryConsumeParam = new ConsumeParam(consumeParam.getRetrySubject(), group, isBroadcast, false, clientId); + this.pullEntry = new PlainPullEntry(consumeParam, pullService, ackService, brokerService, new AlwaysPullStrategy()); + this.retryPullEntry = new PlainPullEntry(retryConsumeParam, pullService, ackService, brokerService, new WeightPullStrategy()); + } + + private static long checkAndGetTimeout(long timeout) { + return timeout < 0 ? timeout : Math.min(Math.max(timeout, MIN_PULL_TIMEOUT_MILLIS), MAX_PULL_TIMEOUT_MILLIS); + } + + @Override + public void online() { + online(CODE); + } + + @Override + public void offline() { + offline(CODE); + } + + public void online(StatusSource src) { + onlineSwitcher.on(src); + LOGGER.info("defaultpullconsumer online. subject={}, group={}", subject(), group()); + } + + public void offline(StatusSource src) { + onlineSwitcher.off(src); + LOGGER.info("defaultpullconsumer offline. subject={}, group={}", subject(), group()); + } + + @Override + public String subject() { + return consumeParam.getSubject(); + } + + @Override + public String group() { + return consumeParam.getGroup(); + } + + @Override + public void setConsumeMostOnce(boolean consumeMostOnce) { + consumeParam.setConsumeMostOnce(consumeMostOnce); + retryConsumeParam.setConsumeMostOnce(consumeMostOnce); + } + + @Override + public boolean isConsumeMostOnce() { + return consumeParam.isConsumeMostOnce(); + } + + @Override + public List pull(int size) { + return newFuture(size, MAX_PULL_TIMEOUT_MILLIS, false).get(); + } + + public List pull(int size, long timeoutMillis) { + return newFuture(size, checkAndGetTimeout(timeoutMillis), false).get(); + } + + @Override + public ListenableFuture> pullFuture(int size) { + return newFuture(size, MAX_PULL_TIMEOUT_MILLIS, false); + } + + @Override + public Future> pullFuture(int size, long timeoutMillis) { + return newFuture(size, checkAndGetTimeout(timeoutMillis), false); + } + + @Override + public Future> pullFuture(int size, long timeoutMillis, boolean isResetCreateTime) { + return newFuture(size, checkAndGetTimeout(timeoutMillis), isResetCreateTime); + } + + abstract PullMessageFuture newFuture(int size, long timeout, boolean isResetCreateTime); +} diff --git a/qmq-client/src/main/java/qunar/tc/qmq/consumer/pull/AbstractPullEntry.java b/qmq-client/src/main/java/qunar/tc/qmq/consumer/pull/AbstractPullEntry.java new file mode 100644 index 00000000..d8d214be --- /dev/null +++ b/qmq-client/src/main/java/qunar/tc/qmq/consumer/pull/AbstractPullEntry.java @@ -0,0 +1,188 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.consumer.pull; + +import com.google.common.base.Strings; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import qunar.tc.qmq.base.BaseMessage; +import qunar.tc.qmq.broker.BrokerGroupInfo; +import qunar.tc.qmq.broker.BrokerService; +import qunar.tc.qmq.common.ClientType; +import qunar.tc.qmq.config.PullSubjectsConfig; +import qunar.tc.qmq.metrics.Metrics; +import qunar.tc.qmq.metrics.QmqCounter; +import qunar.tc.qmq.protocol.CommandCode; +import qunar.tc.qmq.utils.RetrySubjectUtils; + +import java.util.Collections; +import java.util.List; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeoutException; +import java.util.concurrent.atomic.AtomicReference; + +import static qunar.tc.qmq.metrics.MetricsConstants.SUBJECT_GROUP_ARRAY; + +/** + * @author yiqun.fan create on 17-11-2. + */ +abstract class AbstractPullEntry { + private static final Logger LOGGER = LoggerFactory.getLogger(AbstractPullEntry.class); + + private static final int MAX_MESSAGE_RETRY_THRESHOLD = 5; + + private final PullService pullService; + + final BrokerService brokerService; + final AckService ackService; + + private final AtomicReference pullRequestTimeout; + + final WeightLoadBalance loadBalance; + + private final QmqCounter pullWorkCounter; + private final QmqCounter pullFailCounter; + + AbstractPullEntry(String subject, String group, PullService pullService, AckService ackService, BrokerService brokerService) { + this.pullService = pullService; + this.ackService = ackService; + this.brokerService = brokerService; + this.loadBalance = new WeightLoadBalance(); + + pullRequestTimeout = PullSubjectsConfig.get().getPullRequestTimeout(subject); + + String[] values = new String[]{subject, group}; + this.pullWorkCounter = Metrics.counter("qmq_pull_work_count", SUBJECT_GROUP_ARRAY, values); + this.pullFailCounter = Metrics.counter("qmq_pull_fail_count", SUBJECT_GROUP_ARRAY, values); + } + + protected List pull(ConsumeParam consumeParam, BrokerGroupInfo group, int pullSize, int pullTimeout, AckHook ackHook) { + pullWorkCounter.inc(); + AckSendInfo ackSendInfo = ackService.getAckSendInfo(group, consumeParam.getSubject(), consumeParam.getGroup()); + final PullParam pullParam = buildPullParam(consumeParam, group, ackSendInfo, pullSize, pullTimeout); + try { + PullResult pullResult = pullService.pull(pullParam); + List pulledMessages = handlePullResult(pullParam, pullResult, ackHook); + group.markSuccess(); + recordPullSize(group, pulledMessages, pullSize); + return pulledMessages; + } catch (ExecutionException e) { + markFailed(group); + Throwable cause = e.getCause(); + //超时异常暂时不打印日志了 + if (cause instanceof TimeoutException) return Collections.emptyList(); + LOGGER.error("pull message exception. {}", pullParam, e); + } catch (Exception e) { + markFailed(group); + LOGGER.error("pull message exception. {}", pullParam, e); + } + return Collections.emptyList(); + } + + private void markFailed(BrokerGroupInfo group) { + pullFailCounter.inc(); + group.markFailed(); + loadBalance.timeout(group); + } + + private void recordPullSize(BrokerGroupInfo group, List received, int pullSize) { + if (received.size() == 0) { + loadBalance.noMessage(group); + return; + } + + if (received.size() >= pullSize) { + loadBalance.fetchedEnoughMessages(group); + return; + } + + loadBalance.fetchedMessages(group); + } + + private PullParam buildPullParam(ConsumeParam consumeParam, BrokerGroupInfo pullBrokerGroup, AckSendInfo ackSendInfo, int pullSize, int pullTimeout) { + return new PullParam.PullParamBuilder() + .setConsumeParam(consumeParam) + .setBrokerGroup(pullBrokerGroup) + .setPullBatchSize(pullSize) + .setTimeoutMillis(pullTimeout) + .setRequestTimeoutMillis(pullRequestTimeout.get()) + .setMinPullOffset(ackSendInfo.getMinPullOffset()) + .setMaxPullOffset(ackSendInfo.getMaxPullOffset()) + .create(); + } + + private List handlePullResult(final PullParam pullParam, final PullResult pullResult, final AckHook ackHook) { + if (pullResult.getResponseCode() == CommandCode.BROKER_REJECT) { + pullResult.getBrokerGroup().setAvailable(false); + brokerService.refresh(ClientType.CONSUMER, pullParam.getSubject(), pullParam.getGroup()); + } + + List messages = pullResult.getMessages(); + if (messages != null && !messages.isEmpty()) { + monitorMessageCount(pullParam, pullResult); + PulledMessageFilter filter = new PulledMessageFilterImpl(pullParam); + List pulledMessages = ackService.buildPulledMessages(pullParam, pullResult, ackHook, filter); + if (pulledMessages == null || pulledMessages.isEmpty()) { + return Collections.emptyList(); + } + logTimes(pulledMessages); + return pulledMessages; + } + return Collections.emptyList(); + } + + private void logTimes(List pulledMessages) { + for (PulledMessage pulledMessage : pulledMessages) { + int times = pulledMessage.times(); + if (times > MAX_MESSAGE_RETRY_THRESHOLD) { + LOGGER.warn("这是第 {} 次收到同一条消息,请注意检查逻辑是否有问题. subject={}, msgId={}", + times, RetrySubjectUtils.getRealSubject(pulledMessage.getSubject()), pulledMessage.getMessageId()); + } + } + } + + private static void monitorMessageCount(final PullParam pullParam, final PullResult pullResult) { + try { + Metrics.counter("qmq_pull_message_count", new String[]{"subject", "group", "broker"}, + new String[]{pullParam.getSubject(), pullParam.getGroup(), pullParam.getBrokerGroup().getGroupName()}) + .inc(pullResult.getMessages().size()); + } catch (Exception e) { + LOGGER.error("AbstractPullEntry monitor exception", e); + } + } + + private static final class PulledMessageFilterImpl implements PulledMessageFilter { + private final PullParam pullParam; + + PulledMessageFilterImpl(PullParam pullParam) { + this.pullParam = pullParam; + } + + @Override + public boolean filter(PulledMessage message) { + if (pullParam.isConsumeMostOnce() && message.times() > 1) return false; + + //反序列化失败,跳过这个消息 + if (message.getBooleanProperty(BaseMessage.keys.qmq_corruptData.name())) return false; + + // qmq_consumerGroupName + String group = message.getStringProperty(BaseMessage.keys.qmq_consumerGroupName); + return Strings.isNullOrEmpty(group) || group.equals(pullParam.getGroup()); + } + + } +} diff --git a/qmq-client/src/main/java/qunar/tc/qmq/consumer/pull/AckEntry.java b/qmq-client/src/main/java/qunar/tc/qmq/consumer/pull/AckEntry.java new file mode 100644 index 00000000..c92c131c --- /dev/null +++ b/qmq-client/src/main/java/qunar/tc/qmq/consumer/pull/AckEntry.java @@ -0,0 +1,114 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.consumer.pull; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import qunar.tc.qmq.base.BaseMessage; +import qunar.tc.qmq.metrics.Metrics; + +import java.util.concurrent.atomic.AtomicBoolean; + +import static qunar.tc.qmq.metrics.MetricsConstants.SUBJECT_GROUP_ARRAY; + +/** + * @author yiqun.fan create on 17-7-20. + */ +class AckEntry { + private static final Logger LOGGER = LoggerFactory.getLogger(AckEntry.class); + + private final AckSendQueue ackSendQueue; + private final DelayMessageService delayMessageService; + + private final long pullOffset; + + private final AtomicBoolean completing = new AtomicBoolean(false); + private volatile boolean done = false; + private volatile AckEntry next; + + AckEntry(AckSendQueue ackSendQueue, long pullOffset, DelayMessageService delayMessageService) { + this.ackSendQueue = ackSendQueue; + this.pullOffset = pullOffset; + this.delayMessageService = delayMessageService; + } + + void setNext(AckEntry next) { + this.next = next; + } + + AckEntry next() { + return next; + } + + long pullOffset() { + return pullOffset; + } + + public void ack() { + if (!completing.compareAndSet(false, true)) { + return; + } + completed(); + } + + void nack(final int nextRetryCount, final BaseMessage message) { + if (!completing.compareAndSet(false, true)) { + return; + } + doSendNack(nextRetryCount, message); + } + + private void doSendNack(final int nextRetryCount, final BaseMessage message) { + while (true) { + try { + ackSendQueue.sendBackAndCompleteNack(nextRetryCount, message, this); + return; + } catch (Exception e) { + LOGGER.warn("nack exception. subject={}, group={}", ackSendQueue.getSubject(), ackSendQueue.getGroup(), e); + Metrics.counter("qmq_pull_sendNack_error", SUBJECT_GROUP_ARRAY, new String[]{message.getSubject(), ackSendQueue.getGroup()}).inc(); + } + } + } + + void ackDelay(int nextRetryCount, long nextRetryTime, BaseMessage message) { + if (!completing.compareAndSet(false, true)) return; + + try { + if (delayMessageService.sendDelayMessage(nextRetryCount, nextRetryTime, message, ackSendQueue.getGroup())) { + completed(); + LOGGER.info("send delay message: " + message.getMessageId()); + return; + } + Metrics.counter("qmq_pull_sendAckDelay_error", SUBJECT_GROUP_ARRAY, new String[]{message.getSubject(), ackSendQueue.getGroup()}).inc(); + } catch (Exception e) { + LOGGER.error("发送延迟消息失败,改成发送nack. subject={}, messageId={}", message.getSubject(), message.getMessageId(), e); + Metrics.counter("qmq_pull_sendAckDelay_error", SUBJECT_GROUP_ARRAY, new String[]{message.getSubject(), ackSendQueue.getGroup()}).inc(); + } + + + doSendNack(nextRetryCount, message); + } + + void completed() { + done = true; + ackSendQueue.ackCompleted(this); + } + + boolean isDone() { + return done; + } +} \ No newline at end of file diff --git a/qmq-client/src/main/java/qunar/tc/qmq/consumer/pull/AckHelper.java b/qmq-client/src/main/java/qunar/tc/qmq/consumer/pull/AckHelper.java new file mode 100644 index 00000000..09d53e69 --- /dev/null +++ b/qmq-client/src/main/java/qunar/tc/qmq/consumer/pull/AckHelper.java @@ -0,0 +1,72 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.consumer.pull; + +import io.opentracing.Scope; +import io.opentracing.util.GlobalTracer; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import qunar.tc.qmq.NeedRetryException; +import qunar.tc.qmq.base.BaseMessage; +import qunar.tc.qmq.consumer.BaseMessageHandler; +import qunar.tc.qmq.metrics.Metrics; + +import static qunar.tc.qmq.metrics.MetricsConstants.SUBJECT_ARRAY; + +/** + * @author yiqun.fan create on 17-8-19. + */ +class AckHelper { + private static final Logger LOGGER = LoggerFactory.getLogger(AckHelper.class); + + static void ack(PulledMessage message, Throwable throwable) { + BaseMessageHandler.printError(message, throwable); + final AckEntry ackEntry = message.ackEntry(); + if (throwable == null) { + ackEntry.ack(); + return; + } + final BaseMessage ackMsg = new BaseMessage(message); + int nextRetryCount = message.times() + 1; + ackMsg.setProperty(BaseMessage.keys.qmq_times, nextRetryCount); + if (throwable instanceof NeedRetryException) { + ackEntry.ackDelay(nextRetryCount, ((NeedRetryException) throwable).getNext(), ackMsg); + } else { + ackEntry.nack(nextRetryCount, ackMsg); + } + } + + static void ackWithTrace(PulledMessage message, Throwable throwable) { + Scope scope = GlobalTracer.get() + .buildSpan("Qmq.Consume.Ack") + .withTag("subject", message.getSubject()) + .withTag("messageId", message.getMessageId()) + .withTag("consumerGroup", message.getStringProperty(BaseMessage.keys.qmq_consumerGroupName)) + .startActive(true); + try { + ack(message, throwable); + } catch (Exception e) { + scope.span().log("ack_failed"); + LOGGER.error("ack exception.", e); + Metrics.counter("qmq_pull_ackError", SUBJECT_ARRAY, new String[]{message.getSubject()}).inc(); + } finally { + if (scope != null) { + scope.close(); + } + } + } +} diff --git a/qmq-client/src/main/java/qunar/tc/qmq/consumer/pull/AckHook.java b/qmq-client/src/main/java/qunar/tc/qmq/consumer/pull/AckHook.java new file mode 100644 index 00000000..e319cab6 --- /dev/null +++ b/qmq-client/src/main/java/qunar/tc/qmq/consumer/pull/AckHook.java @@ -0,0 +1,24 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.consumer.pull; + +/** + * @author yiqun.fan create on 17-9-11. + */ +public interface AckHook { + void call(PulledMessage message, Throwable throwable); +} diff --git a/qmq-client/src/main/java/qunar/tc/qmq/consumer/pull/AckSendEntry.java b/qmq-client/src/main/java/qunar/tc/qmq/consumer/pull/AckSendEntry.java new file mode 100644 index 00000000..0cbcef4a --- /dev/null +++ b/qmq-client/src/main/java/qunar/tc/qmq/consumer/pull/AckSendEntry.java @@ -0,0 +1,59 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.consumer.pull; + +/** + * @author yiqun.fan create on 17-8-25. + */ +class AckSendEntry { + private final long pullOffsetBegin; + private final long pullOffsetLast; + private final boolean isBroadcast; + + public AckSendEntry() { + this.pullOffsetBegin = -1; + this.pullOffsetLast = -1; + this.isBroadcast = false; + } + + public AckSendEntry(AckEntry first, AckEntry last, boolean isBroadcast) { + this(first.pullOffset(), last.pullOffset(), isBroadcast); + } + + public AckSendEntry(long pullOffsetBegin, long pullOffsetLast, boolean isBroadcast) { + this.pullOffsetBegin = pullOffsetBegin; + this.pullOffsetLast = pullOffsetLast; + this.isBroadcast = isBroadcast; + } + + public long getPullOffsetBegin() { + return pullOffsetBegin; + } + + public long getPullOffsetLast() { + return pullOffsetLast; + } + + public boolean isBroadcast() { + return isBroadcast; + } + + @Override + public String toString() { + return "[" + pullOffsetBegin + ", " + pullOffsetLast + "]"; + } +} diff --git a/qmq-client/src/main/java/qunar/tc/qmq/consumer/pull/AckSendInfo.java b/qmq-client/src/main/java/qunar/tc/qmq/consumer/pull/AckSendInfo.java new file mode 100644 index 00000000..547a3d92 --- /dev/null +++ b/qmq-client/src/main/java/qunar/tc/qmq/consumer/pull/AckSendInfo.java @@ -0,0 +1,55 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.consumer.pull; + +/** + * @author yiqun.fan create on 17-8-20. + */ +class AckSendInfo { + private int toSendNum = 0; + private long minPullOffset = -1; + private long maxPullOffset = -1; + + public int getToSendNum() { + return toSendNum; + } + + public void setToSendNum(int toSendNum) { + this.toSendNum = toSendNum; + } + + public long getMinPullOffset() { + return minPullOffset; + } + + public void setMinPullOffset(long minPullOffset) { + this.minPullOffset = minPullOffset; + } + + public long getMaxPullOffset() { + return maxPullOffset; + } + + public void setMaxPullOffset(long maxPullOffset) { + this.maxPullOffset = maxPullOffset; + } + + @Override + public String toString() { + return "(" + minPullOffset + ", " + maxPullOffset + ", " + toSendNum + ")"; + } +} diff --git a/qmq-client/src/main/java/qunar/tc/qmq/consumer/pull/AckSendQueue.java b/qmq-client/src/main/java/qunar/tc/qmq/consumer/pull/AckSendQueue.java new file mode 100644 index 00000000..55835647 --- /dev/null +++ b/qmq-client/src/main/java/qunar/tc/qmq/consumer/pull/AckSendQueue.java @@ -0,0 +1,387 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.consumer.pull; + +import com.google.common.base.Strings; +import com.google.common.base.Supplier; +import com.google.common.util.concurrent.RateLimiter; +import io.netty.util.Timeout; +import io.netty.util.TimerTask; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import qunar.tc.qmq.base.BaseMessage; +import qunar.tc.qmq.broker.BrokerGroupInfo; +import qunar.tc.qmq.broker.BrokerService; +import qunar.tc.qmq.common.ClientType; +import qunar.tc.qmq.common.TimerUtil; +import qunar.tc.qmq.config.PullSubjectsConfig; +import qunar.tc.qmq.metrics.Metrics; +import qunar.tc.qmq.metrics.QmqCounter; +import qunar.tc.qmq.metrics.QmqMeter; +import qunar.tc.qmq.utils.RetrySubjectUtils; + +import java.util.List; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.atomic.AtomicReference; +import java.util.concurrent.locks.ReentrantLock; + +import static qunar.tc.qmq.metrics.MetricsConstants.SUBJECT_GROUP_ARRAY; + +/** + * @author yiqun.fan create on 17-8-23. + */ +class AckSendQueue implements TimerTask { + private static final Logger LOGGER = LoggerFactory.getLogger(AckSendQueue.class); + private static final long DEFAULT_PULL_OFFSET = -1; + private static final int ACK_INTERVAL_SECONDS = 10; + private static final long ACK_TRY_SEND_TIMEOUT_MILLIS = 1000; + + private static final int DESTROY_CHECK_WAIT_MILLIS = 50; + + private final String brokerGroupName; + private final String subject; + private final String group; + + private final AckService ackService; + + private final String retrySubject; + private final String deadRetrySubject; + + private final AtomicReference pullBatchSize; + + private final ReentrantLock updateLock = new ReentrantLock(); + + private final AtomicLong minPullOffset = new AtomicLong(DEFAULT_PULL_OFFSET); + private final AtomicLong maxPullOffset = new AtomicLong(DEFAULT_PULL_OFFSET); + + private final AtomicInteger toSendNum = new AtomicInteger(0); + private QmqMeter sendNumQps; + private QmqCounter appendErrorCount; + private QmqCounter sendErrorCount; + private QmqCounter sendFailCount; + private QmqCounter deadQueueCount; + + private final LinkedBlockingQueue sendEntryQueue = new LinkedBlockingQueue<>(); + private final ReentrantLock sendLock = new ReentrantLock(); + private final AtomicBoolean inSending = new AtomicBoolean(false); + + private final BrokerService brokerService; + private final SendMessageBack sendMessageBack; + private final boolean isBroadcast; + + private final RateLimiter ackSendFailLogLimit = RateLimiter.create(0.5); + + private volatile AckEntry head = null; + private volatile AckEntry tail = null; + private volatile AckEntry beginScanPosition = null; + + private volatile long lastAppendOffset = -1; + private volatile long lastSendOkOffset = -1; + + AckSendQueue(String brokerGroupName, String subject, String group, AckService ackService, BrokerService brokerService, SendMessageBack sendMessageBack, boolean isBroadcast) { + this.brokerGroupName = brokerGroupName; + this.subject = subject; + this.group = group; + this.ackService = ackService; + this.brokerService = brokerService; + this.sendMessageBack = sendMessageBack; + this.isBroadcast = isBroadcast; + + String realSubject = RetrySubjectUtils.isRetrySubject(subject) ? RetrySubjectUtils.getRealSubject(subject) : subject; + this.retrySubject = RetrySubjectUtils.buildRetrySubject(realSubject, group); + this.deadRetrySubject = RetrySubjectUtils.buildDeadRetrySubject(realSubject, group); + this.pullBatchSize = PullSubjectsConfig.get().getPullBatchSize(subject); + } + + void append(final List batch) { + if (batch == null || batch.isEmpty()) return; + + updateLock.lock(); + try { + if (lastAppendOffset != -1 && lastAppendOffset + 1 != batch.get(0).pullOffset()) { + LOGGER.warn("{}/{} append ack entry not continous. last: {}, new: {}", subject, group, lastAppendOffset, batch.get(0).pullOffset()); + appendErrorCount.inc(); + } + + if (head == null) { + beginScanPosition = head = batch.get(0); + minPullOffset.set(head.pullOffset()); + } + + if (tail != null) { + tail.setNext(batch.get(0)); + } + + tail = batch.get(batch.size() - 1); + lastAppendOffset = tail.pullOffset(); + maxPullOffset.set(tail.pullOffset()); + toSendNum.getAndAdd(batch.size()); + } finally { + updateLock.unlock(); + } + } + + void sendBackAndCompleteNack(final int nextRetryCount, final BaseMessage message, final AckEntry ackEntry) { + final String sendSubject = nextRetryCount > message.getMaxRetryNum() ? deadRetrySubject : retrySubject; + if (deadRetrySubject.equals(sendSubject)) { + deadQueueCount.inc(); + LOGGER.warn("process message retry num {} >= {}, and dead retry. subject={}, group={}, msgId={}", + nextRetryCount - 1, message.getMaxRetryNum(), subject, group, message.getMessageId()); + } + message.setSubject(sendSubject); + sendMessageBack.sendBackAndCompleteNack(nextRetryCount, message, ackEntry); + } + + void ackCompleted(AckEntry current) { + if (current == null) return; + + updateLock.lock(); + try { + if (beginScanPosition == null || beginScanPosition.pullOffset() != current.pullOffset()) { + return; + } + + AckEntry end = scanCompleted(current); + beginScanPosition = end.next(); + if (allowSendAck(end)) { + final AckSendEntry sendEntry = new AckSendEntry(head, end, isBroadcast); + head = beginScanPosition; + if (head == null) { + tail = null; + } + sendEntryQueue.offer(sendEntry); + } else { + return; + } + } finally { + updateLock.unlock(); + } + sendAck(); + } + + private boolean allowSendAck(AckEntry needAck) { + return needAck.next() == null || needAck.pullOffset() - head.pullOffset() >= pullBatchSize.get() - 1; + } + + private AckEntry scanCompleted(AckEntry begin) { + AckEntry needAck = begin; + while (needAck.next() != null && needAck.next().isDone()) { + needAck = needAck.next(); + } + return needAck; + } + + boolean trySendAck(long timeout) { + if (!tryLock(timeout)) return false; + + try { + if (head == null || !head.isDone()) { + return sendAck(); + } + + AckEntry end = scanCompleted(head); + + final AckSendEntry sendEntry = new AckSendEntry(head, end, isBroadcast); + head = beginScanPosition = end.next(); + + if (head == null) { + tail = null; + } + sendEntryQueue.offer(sendEntry); + } finally { + updateLock.unlock(); + } + return sendAck(); + } + + private boolean tryLock(long timeout) { + try { + if (!updateLock.tryLock(timeout, TimeUnit.MILLISECONDS)) { + return false; + } + } catch (InterruptedException e) { + return false; + } + return true; + } + + private boolean sendAck() { + AckSendEntry sendEntry; + + if (inSending.get()) return false; + sendLock.lock(); + try { + if (inSending.get() || sendEntryQueue.isEmpty()) return false; + sendEntry = sendEntryQueue.peek(); + if (sendEntry != null) { + inSending.set(true); + } else { + sendEntryQueue.poll(); + LOGGER.error("sendEntry is null"); + return false; + } + } finally { + sendLock.unlock(); + } + + doSendAck(sendEntry); + return true; + } + + private void doSendAck(final AckSendEntry sendEntry) { + BrokerGroupInfo brokerGroup = getBrokerGroup(); + if (brokerGroup == null) { + LOGGER.debug("lost broker group: {}. subject={}, consumeGroup={}", brokerGroupName, subject, group); + inSending.set(false); + return; + } + + ackService.sendAck(brokerGroup, subject, group, sendEntry, new AckService.SendAckCallback() { + @Override + public void success() { + if (lastSendOkOffset != -1 && lastSendOkOffset + 1 != sendEntry.getPullOffsetBegin()) { + LOGGER.warn("{}/{} ack send not continous. last={}, send={}", subject, group, lastSendOkOffset, sendEntry); + sendErrorCount.inc(); + } + lastSendOkOffset = sendEntry.getPullOffsetLast(); + + minPullOffset.set(sendEntry.getPullOffsetLast() + 1); + final int sendNum = (int) (sendEntry.getPullOffsetLast() - sendEntry.getPullOffsetBegin()) + 1; + toSendNum.getAndAdd(-sendNum); + sendNumQps.mark(sendNum); + + AckSendEntry head = sendEntryQueue.peek(); + if (head == null || head.getPullOffsetBegin() != sendEntry.getPullOffsetBegin()) { + LOGGER.error("ack send error: {}, {}", sendEntry, head); + sendErrorCount.inc(); + } else { + LOGGER.debug("AckSendRet", "ok [{}, {}]", sendEntry.getPullOffsetBegin(), sendEntry.getPullOffsetLast()); + sendEntryQueue.poll(); + } + + inSending.set(false); + AckSendQueue.this.sendAck(); + } + + @Override + public void fail(Exception ex) { + if (ackSendFailLogLimit.tryAcquire()) { + LOGGER.warn("send ack fail, will retry next", ex); + } + LOGGER.debug("AckSendRet", "fail [{}, {}]", sendEntry.getPullOffsetBegin(), sendEntry.getPullOffsetLast()); + sendFailCount.inc(); + inSending.set(false); + } + }); + } + + private BrokerGroupInfo getBrokerGroup() { + return brokerService.getClusterBySubject(ClientType.CONSUMER, subject, group).getGroupByName(brokerGroupName); + } + + AckSendInfo getAckSendInfo() { + AckSendInfo info = new AckSendInfo(); + info.setMinPullOffset(minPullOffset.get()); + info.setMaxPullOffset(maxPullOffset.get()); + info.setToSendNum(toSendNum.get()); + return info; + } + + + void init() { + TimerUtil.newTimeout(this, ACK_INTERVAL_SECONDS, TimeUnit.SECONDS); + + String[] values = new String[]{subject, group}; + sendNumQps = Metrics.meter("qmq_pull_ack_sendnum_qps", SUBJECT_GROUP_ARRAY, values); + appendErrorCount = Metrics.counter("qmq_pull_ack_appenderror_count", SUBJECT_GROUP_ARRAY, values); + sendErrorCount = Metrics.counter("qmq_pull_ack_senderror_count", SUBJECT_GROUP_ARRAY, values); + sendFailCount = Metrics.counter("qmq_pull_ack_sendfail_count", SUBJECT_GROUP_ARRAY, values); + deadQueueCount = Metrics.counter("qmq_deadqueue_send_count", SUBJECT_GROUP_ARRAY, values); + + Metrics.gauge("qmq_pull_ack_min_offset", SUBJECT_GROUP_ARRAY, values, new Supplier() { + @Override + public Double get() { + return (double) minPullOffset.get(); + } + }); + Metrics.gauge("qmq_pull_ack_max_offset", SUBJECT_GROUP_ARRAY, values, new Supplier() { + @Override + public Double get() { + return (double) maxPullOffset.get(); + } + }); + Metrics.gauge("qmq_pull_ack_tosendnum", SUBJECT_GROUP_ARRAY, values, new Supplier() { + @Override + public Double get() { + return (double) toSendNum.get(); + } + }); + } + + public String getSubject() { + return subject; + } + + public String getGroup() { + return group; + } + + private static final AckSendEntry EMPTY_ACK = new AckSendEntry(); + + private static final AckService.SendAckCallback EMPTY_ACK_CALLBACK = new AckService.SendAckCallback() { + @Override + public void success() { + LOGGER.info("send empty Ack ok"); + } + + @Override + public void fail(Exception ex) { + LOGGER.error("send empty Ack fail: " + Strings.nullToEmpty(ex.getMessage())); + } + }; + + @Override + public void run(Timeout timeout) { + try { + if (!trySendAck(ACK_TRY_SEND_TIMEOUT_MILLIS)) { + final BrokerGroupInfo brokerGroup = getBrokerGroup(); + if (brokerGroup == null) { + LOGGER.debug("lost broker group: {}. subject={}, consumeGroup={}", brokerGroupName, subject, group); + return; + } + ackService.sendAck(brokerGroup, subject, group, EMPTY_ACK, EMPTY_ACK_CALLBACK); + } + } finally { + TimerUtil.newTimeout(this, ACK_INTERVAL_SECONDS, TimeUnit.SECONDS); + } + } + + void destroy(long waitTime) { + while (waitTime > 0 && toSendNum.get() > 0) { + try { + Thread.sleep(DESTROY_CHECK_WAIT_MILLIS); + } catch (Exception e) { + break; + } + waitTime -= DESTROY_CHECK_WAIT_MILLIS; + } + } +} diff --git a/qmq-client/src/main/java/qunar/tc/qmq/consumer/pull/AckService.java b/qmq-client/src/main/java/qunar/tc/qmq/consumer/pull/AckService.java new file mode 100644 index 00000000..ef8f0a27 --- /dev/null +++ b/qmq-client/src/main/java/qunar/tc/qmq/consumer/pull/AckService.java @@ -0,0 +1,269 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.consumer.pull; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import qunar.tc.qmq.base.BaseMessage; +import qunar.tc.qmq.broker.BrokerGroupInfo; +import qunar.tc.qmq.broker.BrokerService; +import qunar.tc.qmq.common.ClientType; +import qunar.tc.qmq.common.MapKeyBuilder; +import qunar.tc.qmq.consumer.pull.exception.AckException; +import qunar.tc.qmq.metrics.Metrics; +import qunar.tc.qmq.metrics.QmqCounter; +import qunar.tc.qmq.netty.client.NettyClient; +import qunar.tc.qmq.netty.client.ResponseFuture; +import qunar.tc.qmq.netty.exception.ClientSendException; +import qunar.tc.qmq.protocol.CommandCode; +import qunar.tc.qmq.protocol.Datagram; +import qunar.tc.qmq.protocol.consumer.AckRequest; +import qunar.tc.qmq.protocol.consumer.AckRequestPayloadHolder; +import qunar.tc.qmq.util.RemotingBuilder; + +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.TimeUnit; + +import static qunar.tc.qmq.metrics.MetricsConstants.SUBJECT_GROUP_ARRAY; + +/** + * @author yiqun.fan create on 17-8-18. + */ +class AckService { + private static final Logger LOGGER = LoggerFactory.getLogger(AckService.class); + + private static final QmqCounter GET_PULL_OFFSET_ERROR = Metrics.counter("qmq_pull_getpulloffset_error"); + + private static final long ACK_REQUEST_TIMEOUT_MILLIS = 10 * 1000; + + private final NettyClient client = NettyClient.getClient(); + private final ConcurrentMap senderMap = new ConcurrentHashMap<>(); + + private final BrokerService brokerService; + private final SendMessageBack sendMessageBack; + private final DelayMessageService delayMessageService; + + private String clientId; + + /** + * wait ack send server after client closed + */ + private int destroyWaitInSeconds = 5; + + AckService(BrokerService brokerService) { + this.brokerService = brokerService; + this.sendMessageBack = new SendMessageBackImpl(brokerService); + this.delayMessageService = new DelayMessageService(brokerService, sendMessageBack); + } + + List buildPulledMessages(PullParam pullParam, PullResult pullResult, AckHook ackHook, PulledMessageFilter filter) { + final List pulledMessages = pullResult.getMessages(); + final List result = new ArrayList<>(pulledMessages.size()); + final List ignoreMessages = new ArrayList<>(); + final List ackEntries = new ArrayList<>(pulledMessages.size()); + final AckSendQueue sendQueue = getOrCreateSendQueue(pullResult.getBrokerGroup(), pullParam.getSubject(), pullParam.getGroup(), pullParam.isBroadcast()); + + long prevPullOffset = 0; + AckEntry preAckEntry = null; + + for (BaseMessage message : pulledMessages) { + final long pullOffset = getOffset(message); + if (pullOffset < prevPullOffset) { + monitorGetPullOffsetError(message); + continue; + } + + prevPullOffset = pullOffset; + AckEntry ackEntry = new AckEntry(sendQueue, pullOffset, delayMessageService); + ackEntries.add(ackEntry); + if (preAckEntry != null) { + preAckEntry.setNext(ackEntry); + } + preAckEntry = ackEntry; + + PulledMessage pulledMessage = new PulledMessage(message, ackEntry, ackHook); + if (filter.filter(pulledMessage)) { + result.add(pulledMessage); + } else { + ignoreMessages.add(pulledMessage); + } + + pulledMessage.setSubject(pullParam.getOriginSubject()); + pulledMessage.setProperty(BaseMessage.keys.qmq_consumerGroupName, pullParam.getGroup()); + } + sendQueue.append(ackEntries); + ackIgnoreMessages(ignoreMessages); + preAckOnDemand(result, pullParam.isConsumeMostOnce()); + return result; + } + + private void preAckOnDemand(List messages, boolean isConsumeMostOnce) { + for (PulledMessage message : messages) { + if (isConsumeMostOnce) { + AckHelper.ackWithTrace(message, null); + } + } + } + + private void ackIgnoreMessages(List ignoreMessages) { + for (PulledMessage message : ignoreMessages) { + AckHelper.ackWithTrace(message, null); + } + } + + private AckSendQueue getOrCreateSendQueue(BrokerGroupInfo brokerGroup, String subject, String group, boolean isBroadcast) { + final String senderKey = MapKeyBuilder.buildSenderKey(brokerGroup.getGroupName(), subject, group); + AckSendQueue sender = senderMap.get(senderKey); + if (sender != null) return sender; + + sender = new AckSendQueue(brokerGroup.getGroupName(), subject, group, this, this.brokerService, this.sendMessageBack, isBroadcast); + AckSendQueue old = senderMap.putIfAbsent(senderKey, sender); + if (old == null) { + sender.init(); + return sender; + } + return old; + } + + private long getOffset(BaseMessage message) { + Object offsetObj = message.getProperty(BaseMessage.keys.qmq_pullOffset); + if (offsetObj == null) { + return -1; + } + try { + return Long.parseLong(offsetObj.toString()); + } catch (Exception e) { + return -1; + } + } + + private static void monitorGetPullOffsetError(BaseMessage message) { + LOGGER.error("lost pull offset. msgId=" + message.getMessageId()); + GET_PULL_OFFSET_ERROR.inc(); + } + + void sendAck(BrokerGroupInfo brokerGroup, String subject, String group, AckSendEntry ack, SendAckCallback callback) { + AckRequest request = buildAckRequest(subject, group, ack); + Datagram datagram = RemotingBuilder.buildRequestDatagram(CommandCode.ACK_REQUEST, new AckRequestPayloadHolder(request)); + sendRequest(brokerGroup, subject, group, request, datagram, callback); + } + + private AckRequest buildAckRequest(String subject, String group, AckSendEntry ack) { + AckRequest request = new AckRequest(); + request.setSubject(subject); + request.setGroup(group); + request.setConsumerId(clientId); + request.setPullOffsetBegin(ack.getPullOffsetBegin()); + request.setPullOffsetLast(ack.getPullOffsetLast()); + request.setBroadcast((byte) (ack.isBroadcast() ? 1 : 0)); + return request; + } + + private void sendRequest(BrokerGroupInfo brokerGroup, String subject, String group, AckRequest request, Datagram datagram, SendAckCallback callback) { + try { + client.sendAsync(brokerGroup.getMaster(), datagram, ACK_REQUEST_TIMEOUT_MILLIS, new AckResponseCallback(request, callback, brokerService)); + } catch (ClientSendException e) { + ClientSendException.SendErrorCode errorCode = e.getSendErrorCode(); + monitorAckError(subject, group, errorCode.ordinal()); + callback.fail(e); + } catch (Exception e) { + monitorAckError(subject, group, -1); + callback.fail(e); + } + } + + public void setClientId(String clientId) { + this.clientId = clientId; + } + + public void setDestroyWaitInSeconds(int destroyWaitInSeconds) { + this.destroyWaitInSeconds = destroyWaitInSeconds; + } + + private static final class AckResponseCallback implements ResponseFuture.Callback { + private final AckRequest request; + private final SendAckCallback sendAckCallback; + private final BrokerService brokerService; + + AckResponseCallback(AckRequest request, SendAckCallback sendAckCallback, BrokerService brokerService) { + this.request = request; + this.sendAckCallback = sendAckCallback; + this.brokerService = brokerService; + } + + @Override + public void processResponse(ResponseFuture responseFuture) { + monitorAckTime(request.getSubject(), request.getGroup(), responseFuture.getRequestCostTime()); + + Datagram response = responseFuture.getResponse(); + if (!responseFuture.isSendOk() || response == null) { + monitorAckError(request.getSubject(), request.getGroup(), -1); + sendAckCallback.fail(new AckException("send fail")); + this.brokerService.refresh(ClientType.CONSUMER, request.getSubject(), request.getGroup()); + return; + } + final short responseCode = response.getHeader().getCode(); + if (responseCode == CommandCode.SUCCESS) { + sendAckCallback.success(); + } else { + monitorAckError(request.getSubject(), request.getGroup(), 100 + responseCode); + this.brokerService.refresh(ClientType.CONSUMER, request.getSubject(), request.getGroup()); + sendAckCallback.fail(new AckException("responseCode: " + responseCode)); + } + } + } + + void tryCleanAck() { + for (AckSendQueue sendQueue : senderMap.values()) { + try { + sendQueue.trySendAck(1000); + } catch (Exception e) { + LOGGER.error("try clean ack exception", e); + } + } + } + + void destroy() { + tryCleanAck(); + for (AckSendQueue sendQueue : senderMap.values()) { + sendQueue.destroy(destroyWaitInSeconds * 1000L); + } + } + + private static void monitorAckTime(String subject, String group, long time) { + Metrics.timer("qmq_pull_ack_timer", SUBJECT_GROUP_ARRAY, new String[]{subject, group}).update(time, TimeUnit.MILLISECONDS); + } + + private static void monitorAckError(String subject, String group, int errorCode) { + LOGGER.error("ack error. subject={}, group={}, errorCode={}", subject, group, errorCode); + Metrics.counter("qmq_pull_ack_error", SUBJECT_GROUP_ARRAY, new String[]{subject, group}).inc(); + } + + AckSendInfo getAckSendInfo(BrokerGroupInfo brokerGroup, String subject, String group) { + AckSendQueue sender = senderMap.get(MapKeyBuilder.buildSenderKey(brokerGroup.getGroupName(), subject, group)); + return sender != null ? sender.getAckSendInfo() : new AckSendInfo(); + } + + interface SendAckCallback { + void success(); + + void fail(Exception ex); + } +} diff --git a/qmq-client/src/main/java/qunar/tc/qmq/consumer/pull/AlwaysPullStrategy.java b/qmq-client/src/main/java/qunar/tc/qmq/consumer/pull/AlwaysPullStrategy.java new file mode 100644 index 00000000..9876b44e --- /dev/null +++ b/qmq-client/src/main/java/qunar/tc/qmq/consumer/pull/AlwaysPullStrategy.java @@ -0,0 +1,29 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.consumer.pull; + +class AlwaysPullStrategy implements PullStrategy { + @Override + public boolean needPull() { + return true; + } + + @Override + public void record(boolean status) { + + } +} diff --git a/qmq-client/src/main/java/qunar/tc/qmq/consumer/pull/ConsumeParam.java b/qmq-client/src/main/java/qunar/tc/qmq/consumer/pull/ConsumeParam.java new file mode 100644 index 00000000..a65d4e77 --- /dev/null +++ b/qmq-client/src/main/java/qunar/tc/qmq/consumer/pull/ConsumeParam.java @@ -0,0 +1,107 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.consumer.pull; + +import qunar.tc.qmq.TagType; +import qunar.tc.qmq.consumer.register.RegistParam; +import qunar.tc.qmq.utils.RetrySubjectUtils; + +import java.util.Collections; +import java.util.Set; + +/** + * @author yiqun.fan create on 17-11-2. + */ +class ConsumeParam { + private final String subject; + private final String group; + private final String originSubject; + private final String retrySubject; + private final String consumerId; + private final boolean isBroadcast; + private volatile boolean isConsumeMostOnce; + private volatile TagType tagType; + private volatile Set tags; + + public ConsumeParam(String subject, String group, RegistParam param) { + this(subject, group, param.isBroadcast(), param.getSubscribeParam().isConsumeMostOnce(), param.getSubscribeParam().getTagType(), param.getSubscribeParam().getTags(), param.getClientId()); + } + + public ConsumeParam(String subject, String group, boolean isBroadcast, boolean isConsumeMostOnce, String clientId) { + this(subject, group, isBroadcast, isConsumeMostOnce, TagType.NO_TAG, Collections.EMPTY_SET, clientId); + } + + public ConsumeParam(String subject, String group, boolean isBroadcast, boolean isConsumeMostOnce, TagType tagType, Set tags, String clientId) { + this.subject = subject; + this.group = group; + this.originSubject = RetrySubjectUtils.isRetrySubject(subject) ? RetrySubjectUtils.getRealSubject(subject) : subject; + this.retrySubject = RetrySubjectUtils.buildRetrySubject(originSubject, group); + this.consumerId = clientId; + this.isBroadcast = isBroadcast; + this.isConsumeMostOnce = isConsumeMostOnce; + this.tagType = tagType; + this.tags = tags; + } + + public String getSubject() { + return subject; + } + + public String getGroup() { + return group; + } + + public String getOriginSubject() { + return originSubject; + } + + public String getRetrySubject() { + return retrySubject; + } + + public String getConsumerId() { + return consumerId; + } + + public boolean isBroadcast() { + return isBroadcast; + } + + public boolean isConsumeMostOnce() { + return isConsumeMostOnce; + } + + public void setConsumeMostOnce(boolean consumeMostOnce) { + isConsumeMostOnce = consumeMostOnce; + } + + public TagType getTagType() { + return tagType; + } + + public void setTagType(TagType tagType) { + this.tagType = tagType; + } + + public Set getTags() { + return tags; + } + + public void setTags(Set tags) { + this.tags = tags; + } +} diff --git a/qmq-client/src/main/java/qunar/tc/qmq/consumer/pull/DefaultPullConsumer.java b/qmq-client/src/main/java/qunar/tc/qmq/consumer/pull/DefaultPullConsumer.java new file mode 100644 index 00000000..5541cc26 --- /dev/null +++ b/qmq-client/src/main/java/qunar/tc/qmq/consumer/pull/DefaultPullConsumer.java @@ -0,0 +1,192 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.consumer.pull; + +import com.google.common.collect.Lists; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import qunar.tc.qmq.Message; +import qunar.tc.qmq.broker.BrokerService; +import qunar.tc.qmq.config.PullSubjectsConfig; +import qunar.tc.qmq.metrics.Metrics; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.TimeUnit; + +import static qunar.tc.qmq.metrics.MetricsConstants.SUBJECT_GROUP_ARRAY; + +/** + * @author yiqun.fan create on 17-9-20. + */ +class DefaultPullConsumer extends AbstractPullConsumer implements Runnable { + private static final Logger LOGGER = LoggerFactory.getLogger(DefaultPullConsumer.class); + + private static final int POLL_TIMEOUT_MILLIS = 1000; + private static final int MAX_FETCH_SIZE = 10000; + private static final int MAX_TIMEOUT = 5 * 60 * 1000; + + private volatile boolean isStop = false; + + private final LinkedBlockingQueue requestQueue = new LinkedBlockingQueue<>(); + private final LinkedBlockingQueue localBuffer = new LinkedBlockingQueue<>(); + + private final int preFetchSize; + private final int lowWaterMark; + + DefaultPullConsumer(String subject, String group, boolean isBroadcast, String clientId, PullService pullService, AckService ackService, BrokerService brokerService) { + super(subject, group, isBroadcast, clientId, pullService, ackService, brokerService); + this.preFetchSize = PullSubjectsConfig.get().getPullBatchSize(subject).get(); + this.lowWaterMark = Math.round(preFetchSize * 0.2F); + } + + @Override + PullMessageFuture newFuture(int size, long timeout, boolean isResetCreateTime) { + if (size > MAX_FETCH_SIZE) { + size = MAX_FETCH_SIZE; + } + if (timeout > MAX_TIMEOUT) { + timeout = MAX_TIMEOUT; + } + + if (size <= 0) { + PullMessageFuture future = new PullMessageFuture(size, size, timeout, isResetCreateTime); + future.set(Collections.emptyList()); + return future; + } + + if (localBuffer.size() == 0) { + int fetchSize = Math.max(size, preFetchSize); + PullMessageFuture future = new PullMessageFuture(size, fetchSize, timeout, isResetCreateTime); + requestQueue.offer(future); + return future; + } + + if (localBuffer.size() >= size) { + List messages = new ArrayList<>(size); + localBuffer.drainTo(messages, size); + if (messages.size() > 0) { + checkLowWaterMark(); + PullMessageFuture future = new PullMessageFuture(size, size, timeout, isResetCreateTime); + future.set(messages); + return future; + } + } + int fetchSize = Math.max(preFetchSize, size - localBuffer.size()); + PullMessageFuture future = new PullMessageFuture(size, fetchSize, timeout, isResetCreateTime); + requestQueue.offer(future); + return future; + } + + private void checkLowWaterMark() { + int bufferSize = localBuffer.size(); + if (bufferSize < lowWaterMark) { + int fetchSize = preFetchSize - bufferSize; + PullMessageFuture future = new PullMessageFuture(0, fetchSize, -1, false); + requestQueue.offer(future); + } + } + + @Override + public void run() { + PullMessageFuture future; + while (!isStop) { + try { + if (!onlineSwitcher.waitOn()) continue; + + future = requestQueue.poll(POLL_TIMEOUT_MILLIS, TimeUnit.MILLISECONDS); + if (future != null) doPull(future); + } catch (InterruptedException e) { + LOGGER.error("pullConsumer poll be interrupted. subject={}, group={}", subject(), group(), e); + } catch (Exception e) { + LOGGER.error("pullConsumer poll exception. subject={}, group={}", subject(), group(), e); + } + } + } + + private void doPull(PullMessageFuture request) { + List messages = Lists.newArrayListWithCapacity(request.getFetchSize()); + try { + retryPullEntry.pull(request.getFetchSize(), request.getTimeout(), messages); + if (messages.size() > 0 && request.isPullOnce()) return; + + if (request.isResetCreateTime()) { + request.resetCreateTime(); + } + + do { + int fetchSize = request.getFetchSize() - messages.size(); + if (fetchSize <= 0) break; + PlainPullEntry.PlainPullResult result = pullEntry.pull(fetchSize, request.getTimeout(), messages); + if (result == PlainPullEntry.PlainPullResult.NO_BROKER) { + break; + } + } while (messages.size() < request.getFetchSize() && !request.isExpired()); + } catch (Exception e) { + LOGGER.error("DefaultPullConsumer doPull exception. subject={}, group={}", subject(), group(), e); + Metrics.counter("qmq_pull_defaultPull_doPull_fail", SUBJECT_GROUP_ARRAY, new String[]{subject(), group()}).inc(); + } finally { + setResult(request, messages); + } + } + + private void setResult(PullMessageFuture future, List messages) { + int expectedSize = future.getExpectedSize(); + if (expectedSize <= 0) { + localBuffer.addAll(messages); + future.set(Collections.emptyList()); + return; + } + + List result = new ArrayList<>(expectedSize); + int bufferSize = localBuffer.size(); + if (bufferSize > 0) { + localBuffer.drainTo(result, Math.min(expectedSize, bufferSize)); + } + int need = expectedSize - result.size(); + if (need <= 0) { + localBuffer.addAll(messages); + future.set(result); + return; + } + + result.addAll(head(messages, need)); + localBuffer.addAll(tail(messages, need)); + future.set(result); + } + + private List head(List input, int head) { + if (head >= input.size()) { + return input; + } + return input.subList(0, head); + } + + private List tail(List input, int head) { + if (head >= input.size()) { + return Collections.emptyList(); + } + return input.subList(head, input.size()); + } + + @Override + public void close() { + isStop = true; + } +} diff --git a/qmq-client/src/main/java/qunar/tc/qmq/consumer/pull/DelayMessageService.java b/qmq-client/src/main/java/qunar/tc/qmq/consumer/pull/DelayMessageService.java new file mode 100644 index 00000000..b757c6af --- /dev/null +++ b/qmq-client/src/main/java/qunar/tc/qmq/consumer/pull/DelayMessageService.java @@ -0,0 +1,122 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.consumer.pull; + +import com.google.common.util.concurrent.SettableFuture; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import qunar.tc.qmq.base.BaseMessage; +import qunar.tc.qmq.broker.BrokerClusterInfo; +import qunar.tc.qmq.broker.BrokerGroupInfo; +import qunar.tc.qmq.broker.BrokerLoadBalance; +import qunar.tc.qmq.broker.BrokerService; +import qunar.tc.qmq.broker.impl.PollBrokerLoadBalance; +import qunar.tc.qmq.common.ClientType; +import qunar.tc.qmq.consumer.pull.exception.SendMessageBackException; +import qunar.tc.qmq.service.exceptions.MessageException; + +import java.util.Date; +import java.util.List; + +/** + * @author yiqun.fan create on 17-10-11. + */ +class DelayMessageService { + private static final Logger LOGGER = LoggerFactory.getLogger(DelayMessageService.class); + + private static final int SEND_SUCCESS = 1; + private static final int SEND_FAIL = -1; + private static final int NO_BROKER = 0; + + private final BrokerService brokerService; + private final SendMessageBack sendMessageBack; + private final BrokerLoadBalance brokerLoadBalance = PollBrokerLoadBalance.getInstance(); + + DelayMessageService(BrokerService brokerService, SendMessageBack sendMessageBack) { + this.brokerService = brokerService; + this.sendMessageBack = sendMessageBack; + } + + boolean sendDelayMessage(int nextRetryCount, long nextRetryTime, BaseMessage message, String group) throws MessageException { + if (!needSendDelay(nextRetryCount, message)) return false; + message.setProperty(BaseMessage.keys.qmq_consumerGroupName, group); + message.setDelayTime(new Date(nextRetryTime)); + return send(message) == SEND_SUCCESS; + } + + /** + * 500 ms以下的不走delay message + * + * @param nextRetryCount + * @param message + * @return + */ + private boolean needSendDelay(int nextRetryCount, BaseMessage message) { + return message.getMaxRetryNum() >= nextRetryCount; + } + + private int send(BaseMessage message) { + BrokerClusterInfo brokerCluster = brokerService.getClusterBySubject(ClientType.DELAY_PRODUCER, message.getSubject()); + List groups = brokerCluster.getGroups(); + if (groups == null || groups.isEmpty()) return NO_BROKER; + + BrokerGroupInfo lastSentBrokerGroup = null; + int result = SEND_FAIL; + for (int i = 0; i < groups.size(); i++) { + try { + BrokerGroupInfo brokerGroup = brokerLoadBalance.loadBalance(brokerCluster, lastSentBrokerGroup); + result = doSend(message, brokerGroup); + lastSentBrokerGroup = brokerGroup; + if (SEND_SUCCESS == result) return result; + + LOGGER.warn("retry send delay message. {}", result); + } catch (Exception e) { + LOGGER.warn("retry send delay message. {}", e.getMessage()); + } + } + return result; + } + + private int doSend(BaseMessage message, BrokerGroupInfo brokerGroup) { + final SettableFuture sendFuture = SettableFuture.create(); + final SendMessageBack.Callback callback = new SendMessageBack.Callback() { + + @Override + public void success() { + sendFuture.set(SEND_SUCCESS); + } + + @Override + public void fail(Throwable e) { + if (e instanceof SendMessageBackException) { + LOGGER.warn("send delay message fail. exception: {}", e.getMessage()); + } else { + LOGGER.warn("send delay message fail", e); + } + sendFuture.set(SEND_FAIL); + } + }; + sendMessageBack.sendBack(brokerGroup, message, callback, ClientType.DELAY_PRODUCER); + try { + return sendFuture.get(); + } catch (Exception e) { + brokerService.refresh(ClientType.DELAY_PRODUCER, message.getSubject()); + LOGGER.warn("send delay message fail. {}", e.getMessage()); + return SEND_FAIL; + } + } +} diff --git a/qmq-client/src/main/java/qunar/tc/qmq/consumer/pull/PlainPullEntry.java b/qmq-client/src/main/java/qunar/tc/qmq/consumer/pull/PlainPullEntry.java new file mode 100644 index 00000000..fd2a9691 --- /dev/null +++ b/qmq-client/src/main/java/qunar/tc/qmq/consumer/pull/PlainPullEntry.java @@ -0,0 +1,61 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.consumer.pull; + +import qunar.tc.qmq.Message; +import qunar.tc.qmq.broker.BrokerClusterInfo; +import qunar.tc.qmq.broker.BrokerGroupInfo; +import qunar.tc.qmq.broker.BrokerService; +import qunar.tc.qmq.common.ClientType; + +import java.util.List; + +/** + * @author yiqun.fan create on 17-9-21. + */ +class PlainPullEntry extends AbstractPullEntry { + + private final ConsumeParam consumeParam; + private final PullStrategy pullStrategy; + + PlainPullEntry(ConsumeParam consumeParam, PullService pullService, AckService ackService, BrokerService brokerService, PullStrategy pullStrategy) { + super(consumeParam.getSubject(), consumeParam.getGroup(), pullService, ackService, brokerService); + this.consumeParam = consumeParam; + this.pullStrategy = pullStrategy; + } + + PlainPullResult pull(final int fetchSize, final int pullTimeout, final List output) { + if (!pullStrategy.needPull()) return PlainPullResult.NOMORE_MESSAGE; + BrokerClusterInfo brokerCluster = brokerService.getClusterBySubject(ClientType.CONSUMER, consumeParam.getSubject(), consumeParam.getGroup()); + List groups = brokerCluster.getGroups(); + if (groups.isEmpty()) { + return PlainPullResult.NO_BROKER; + } + BrokerGroupInfo group = loadBalance.select(brokerCluster); + List received = pull(consumeParam, group, fetchSize, pullTimeout, null); + output.addAll(received); + pullStrategy.record(received.size() > 0); + return PlainPullResult.NOMORE_MESSAGE; + } + + enum PlainPullResult { + NO_BROKER, + NOMORE_MESSAGE, + COMPLETE, + REQUESTING + } +} diff --git a/qmq-client/src/main/java/qunar/tc/qmq/consumer/pull/PullConsumerFactory.java b/qmq-client/src/main/java/qunar/tc/qmq/consumer/pull/PullConsumerFactory.java new file mode 100644 index 00000000..bc1ac509 --- /dev/null +++ b/qmq-client/src/main/java/qunar/tc/qmq/consumer/pull/PullConsumerFactory.java @@ -0,0 +1,72 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.consumer.pull; + +import qunar.tc.qmq.PullConsumer; +import qunar.tc.qmq.common.MapKeyBuilder; +import qunar.tc.qmq.consumer.exception.CreatePullConsumerException; +import qunar.tc.qmq.consumer.exception.DuplicateListenerException; +import qunar.tc.qmq.utils.RetrySubjectUtils; + +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.locks.ReentrantLock; + +/** + * @author yiqun.fan create on 17-9-12. + */ +public class PullConsumerFactory { + private final ConcurrentMap pullConsumerMap = new ConcurrentHashMap<>(); + + private final ReentrantLock pullConsumerMapLock = new ReentrantLock(); + + private final PullRegister pullRegister; + + public PullConsumerFactory(PullRegister pullRegister) { + this.pullRegister = pullRegister; + } + + public PullConsumer getOrCreateDefault(String subject, String group, boolean isBroadcast) { + final String realSubject = RetrySubjectUtils.getRealSubject(subject); + final String key = MapKeyBuilder.buildSubscribeKey(realSubject, group); + PullConsumer consumer = pullConsumerMap.get(key); + if (consumer != null) { + return consumer; + } + pullConsumerMapLock.lock(); + try { + consumer = pullConsumerMap.get(key); + if (consumer != null) { + return consumer; + } + PullConsumer consumerImpl = createDefaultPullConsumer(realSubject, group, isBroadcast); + pullConsumerMap.put(key, consumerImpl); + return consumerImpl; + } catch (Exception e) { + if (e instanceof DuplicateListenerException) { + throw new CreatePullConsumerException("已经使用了onMessage方式处理的主题不能再纯拉模式", realSubject, group); + } + throw e; + } finally { + pullConsumerMapLock.unlock(); + } + } + + private DefaultPullConsumer createDefaultPullConsumer(String subject, String group, boolean isBroadcast) { + return pullRegister.createDefaultPullConsumer(subject, group, isBroadcast); + } +} diff --git a/qmq-client/src/main/java/qunar/tc/qmq/consumer/pull/PullEntry.java b/qmq-client/src/main/java/qunar/tc/qmq/consumer/pull/PullEntry.java new file mode 100644 index 00000000..c5f6d1d0 --- /dev/null +++ b/qmq-client/src/main/java/qunar/tc/qmq/consumer/pull/PullEntry.java @@ -0,0 +1,222 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.consumer.pull; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import qunar.tc.qmq.broker.BrokerClusterInfo; +import qunar.tc.qmq.broker.BrokerGroupInfo; +import qunar.tc.qmq.broker.BrokerService; +import qunar.tc.qmq.common.ClientType; +import qunar.tc.qmq.common.StatusSource; +import qunar.tc.qmq.common.SwitchWaiter; +import qunar.tc.qmq.config.PullSubjectsConfig; +import qunar.tc.qmq.metrics.Metrics; +import qunar.tc.qmq.metrics.QmqCounter; + +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicReference; + +import static qunar.tc.qmq.metrics.MetricsConstants.SUBJECT_GROUP_ARRAY; + +/** + * @author yiqun.fan create on 17-8-18. + */ +class PullEntry extends AbstractPullEntry implements Runnable { + private static final Logger LOGGER = LoggerFactory.getLogger(PullEntry.class); + + static final PullEntry EMPTY_PULL_ENTRY = new PullEntry() { + @Override + void online(StatusSource src) { + } + + @Override + void offline(StatusSource src) { + } + + @Override + public void run() { + } + }; + + private static final long PAUSETIME_OF_CLEAN_LAST_MESSAGE = 200; + private static final long PAUSETIME_OF_NOAVAILABLE_BROKER = 100; + private static final long PAUSETIME_OF_NOMESSAGE = 500; + + private final PushConsumer pushConsumer; + private final AtomicReference pullBatchSize; + private final AtomicReference pullTimeout; + private final AtomicReference ackNosendLimit; + + private final Set brokersOfWaitAck = new HashSet<>(); + + private final AtomicBoolean isRunning = new AtomicBoolean(true); + private final SwitchWaiter onlineSwitcher = new SwitchWaiter(false); + private final QmqCounter pullRunCounter; + private final QmqCounter pauseCounter; + private final String logType; + private final PullStrategy pullStrategy; + + private PullEntry() { + super("", "", null, null, null); + pushConsumer = null; + pullBatchSize = pullTimeout = ackNosendLimit = null; + pullRunCounter = null; + pauseCounter = null; + logType = "PullEntry="; + pullStrategy = null; + } + + PullEntry(PushConsumer pushConsumer, PullService pullService, AckService ackService, BrokerService brokerService, PullStrategy pullStrategy) { + super(pushConsumer.subject(), pushConsumer.group(), pullService, ackService, brokerService); + String subject = pushConsumer.subject(); + String group = pushConsumer.group(); + this.pushConsumer = pushConsumer; + this.pullBatchSize = PullSubjectsConfig.get().getPullBatchSize(subject); + this.pullTimeout = PullSubjectsConfig.get().getPullTimeout(subject); + this.ackNosendLimit = PullSubjectsConfig.get().getAckNosendLimit(subject); + this.pullStrategy = pullStrategy; + + String[] values = new String[]{subject, group}; + this.pullRunCounter = Metrics.counter("qmq_pull_run_count", SUBJECT_GROUP_ARRAY, values); + this.pauseCounter = Metrics.counter("qmq_pull_pause_count", SUBJECT_GROUP_ARRAY, values); + + this.logType = "PullEntry=" + subject; + } + + void online(StatusSource src) { + onlineSwitcher.on(src); + LOGGER.info("pullconsumer online. subject={}, group={}", pushConsumer.subject(), pushConsumer.group()); + } + + void offline(StatusSource src) { + onlineSwitcher.off(src); + LOGGER.info("pullconsumer offline. subject={}, group={}", pushConsumer.subject(), pushConsumer.group()); + } + + void destroy() { + isRunning.set(false); + } + + @Override + public void run() { + final DoPullParam doPullParam = new DoPullParam(); + + while (isRunning.get()) { + try { + if (!preparePull()) { + LOGGER.debug(logType, "preparePull false. subject={}, group={}", pushConsumer.subject(), pushConsumer.group()); + continue; + } + + if (!resetDoPullParam(doPullParam)) { + LOGGER.debug(logType, "buildDoPullParam false. subject={}, group={}", pushConsumer.subject(), pushConsumer.group()); + continue; + } + + if (isRunning.get() && onlineSwitcher.waitOn()) { + doPull(doPullParam); + } + } catch (Exception e) { + LOGGER.error("PullEntry run exception", e); + } + } + } + + private boolean preparePull() { + pullRunCounter.inc(); + if (!pushConsumer.cleanLocalBuffer()) { + pause("wait consumer", PAUSETIME_OF_CLEAN_LAST_MESSAGE); + return false; + } + + if (!pullStrategy.needPull()) { + pause("wait consumer", PAUSETIME_OF_NOMESSAGE); + return false; + } + return true; + } + + private boolean resetDoPullParam(DoPullParam param) { + while (isRunning.get()) { + param.cluster = getBrokerCluster(); + param.broker = nextPullBrokerGroup(param.cluster); + if (BrokerGroupInfo.isInvalid(param.broker)) { + brokersOfWaitAck.clear(); + pause("noavaliable broker", PAUSETIME_OF_NOAVAILABLE_BROKER); + continue; + } + + param.ackSendInfo = ackService.getAckSendInfo(param.broker, pushConsumer.subject(), pushConsumer.group()); + if (param.ackSendInfo.getToSendNum() <= ackNosendLimit.get()) { + brokersOfWaitAck.clear(); + break; + } + param.ackSendInfo = null; + brokersOfWaitAck.add(param.broker.getGroupName()); + } + return isRunning.get() && param.ackSendInfo != null; + } + + private BrokerClusterInfo getBrokerCluster() { + return brokerService.getClusterBySubject(ClientType.CONSUMER, pushConsumer.subject(), pushConsumer.group()); + } + + private BrokerGroupInfo nextPullBrokerGroup(BrokerClusterInfo cluster) { + //没有分配到brokers + List groups = cluster.getGroups(); + if (groups.isEmpty()) return null; + + final int brokerSize = groups.size(); + for (int cnt = 0; cnt < brokerSize; cnt++) { + BrokerGroupInfo broker = loadBalance.select(cluster); + if (broker == null) return null; + if (brokersOfWaitAck.contains(broker.getGroupName())) continue; + + return broker; + } + // 没有可用的brokers + return null; + } + + private void doPull(DoPullParam param) { + List messages = pull(pushConsumer.consumeParam(), param.broker, pullBatchSize.get(), pullTimeout.get(), pushConsumer); + pullStrategy.record(messages.size() > 0); + pushConsumer.push(messages); + } + + private void pause(String log, long timeMillis) { + final String subject = pushConsumer.subject(); + final String group = pushConsumer.group(); + this.pauseCounter.inc(); + LOGGER.debug(logType, "pull pause {} ms, {}. subject={}, group={}", timeMillis, log, subject, group); + try { + Thread.sleep(timeMillis); + } catch (Exception e) { + LOGGER.info("PullEntry pause exception. log={}", log, e); + } + } + + private static final class DoPullParam { + private volatile BrokerClusterInfo cluster = null; + private volatile BrokerGroupInfo broker = null; + private volatile AckSendInfo ackSendInfo = null; + } +} diff --git a/qmq-client/src/main/java/qunar/tc/qmq/consumer/pull/PullMessageFuture.java b/qmq-client/src/main/java/qunar/tc/qmq/consumer/pull/PullMessageFuture.java new file mode 100644 index 00000000..9537918d --- /dev/null +++ b/qmq-client/src/main/java/qunar/tc/qmq/consumer/pull/PullMessageFuture.java @@ -0,0 +1,118 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.consumer.pull; + +import com.google.common.util.concurrent.AbstractFuture; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import qunar.tc.qmq.Message; + +import java.util.Collections; +import java.util.List; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; + +/** + * @author yiqun.fan create on 17-10-19. + */ +class PullMessageFuture extends AbstractFuture> { + private static final Logger LOGGER = LoggerFactory.getLogger(PullMessageFuture.class); + private final int expectedSize; + private final int fetchSize; + private final long timeout; + private final boolean isResetCreateTime; + private volatile long createTime = System.currentTimeMillis(); + + public PullMessageFuture(int expectedSize, int fetchSize, long timeout, boolean isResetCreateTime) { + this.expectedSize = expectedSize; + this.fetchSize = fetchSize; + this.timeout = timeout; + this.isResetCreateTime = isResetCreateTime; + } + + public int getFetchSize() { + return fetchSize; + } + + public int getExpectedSize() { + return expectedSize; + } + + public boolean isResetCreateTime() { + return isResetCreateTime; + } + + public void resetCreateTime() { + createTime = System.currentTimeMillis(); + } + + public boolean isExpired() { + return System.currentTimeMillis() - timeout > createTime; + } + + public boolean isPullOnce() { + return timeout < 0; + } + + public int getTimeout() { + return (int) timeout; + } + + @Override + public boolean cancel(boolean mayInterruptIfRunning) { + throw new UnsupportedOperationException(); + } + + @Override + public boolean isCancelled() { + return false; + } + + @Override + public boolean set(List value) { + return super.set(value); + } + + @Override + public List get() { + while (true) { + try { + return super.get(); + } catch (InterruptedException e) { + LOGGER.info("ignore interrupt pull"); + } catch (ExecutionException e) { + return Collections.emptyList(); + } + } + } + + @Override + public List get(long timeout, TimeUnit unit) throws TimeoutException, ExecutionException { + long current = System.currentTimeMillis(); + long endTime = current + unit.toMillis(timeout); + while (endTime > current) { + try { + return super.get(endTime - current, TimeUnit.MILLISECONDS); + } catch (InterruptedException e) { + LOGGER.info("ignore interrupt pull"); + } + current = System.currentTimeMillis(); + } + throw new TimeoutException(); + } +} diff --git a/qmq-client/src/main/java/qunar/tc/qmq/consumer/pull/PullParam.java b/qmq-client/src/main/java/qunar/tc/qmq/consumer/pull/PullParam.java new file mode 100644 index 00000000..a2681d18 --- /dev/null +++ b/qmq-client/src/main/java/qunar/tc/qmq/consumer/pull/PullParam.java @@ -0,0 +1,179 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.consumer.pull; + +import qunar.tc.qmq.TagType; +import qunar.tc.qmq.broker.BrokerGroupInfo; + +import java.util.Set; + +/** + * @author yiqun.fan create on 17-8-18. + */ +class PullParam { + private final ConsumeParam consumeParam; + private final BrokerGroupInfo brokerGroup; + private final int pullBatchSize; + private final long timeoutMillis; + private final long requestTimeoutMillis; + private final long consumeOffset; + private final long minPullOffset; + private final long maxPullOffset; + + private PullParam(ConsumeParam consumeParam, BrokerGroupInfo brokerGroup, + int pullBatchSize, long timeoutMillis, long requestTimeoutMillis, + long consumeOffset, long minPullOffset, long maxPullOffset) { + this.consumeParam = consumeParam; + this.brokerGroup = brokerGroup; + this.pullBatchSize = pullBatchSize; + this.timeoutMillis = timeoutMillis; + this.requestTimeoutMillis = requestTimeoutMillis; + this.consumeOffset = consumeOffset; + this.minPullOffset = minPullOffset; + this.maxPullOffset = maxPullOffset; + } + + public String getSubject() { + return consumeParam.getSubject(); + } + + public String getGroup() { + return consumeParam.getGroup(); + } + + public String getOriginSubject() { + return consumeParam.getOriginSubject(); + } + + public BrokerGroupInfo getBrokerGroup() { + return brokerGroup; + } + + public int getPullBatchSize() { + return pullBatchSize; + } + + public long getTimeoutMillis() { + return timeoutMillis; + } + + public long getRequestTimeoutMillis() { + return requestTimeoutMillis; + } + + public long getConsumeOffset() { + return consumeOffset; + } + + public long getMinPullOffset() { + return minPullOffset; + } + + public long getMaxPullOffset() { + return maxPullOffset; + } + + public String getConsumerId() { + return consumeParam.getConsumerId(); + } + + public boolean isBroadcast() { + return consumeParam.isBroadcast(); + } + + public boolean isConsumeMostOnce() { + return consumeParam.isConsumeMostOnce(); + } + + + public Set getTags() { + return consumeParam.getTags(); + } + + public TagType getTagType() { + return consumeParam.getTagType(); + } + + + @Override + public String toString() { + return "PullParam{" + + "consumeParam=" + consumeParam + + ", brokerGroup=" + brokerGroup + + ", pullBatchSize=" + pullBatchSize + + ", timeoutMillis=" + timeoutMillis + + ", consumeOffset=" + consumeOffset + + ", minPullOffset=" + minPullOffset + + ", maxPullOffset=" + maxPullOffset + + '}'; + } + + public static final class PullParamBuilder { + private ConsumeParam consumeParam; + private BrokerGroupInfo brokerGroup; + private int pullBatchSize; + private long timeoutMillis; + private long requestTimeoutMillis; + private long consumeOffset = -1; + private long minPullOffset = -1; + private long maxPullOffset = -1; + + public PullParam create() { + return new PullParam(consumeParam, brokerGroup, pullBatchSize, timeoutMillis, requestTimeoutMillis, consumeOffset, minPullOffset, maxPullOffset); + } + + public PullParamBuilder setConsumeParam(ConsumeParam consumeParam) { + this.consumeParam = consumeParam; + return this; + } + + public PullParamBuilder setBrokerGroup(BrokerGroupInfo brokerGroup) { + this.brokerGroup = brokerGroup; + return this; + } + + public PullParamBuilder setPullBatchSize(int pullBatchSize) { + this.pullBatchSize = pullBatchSize; + return this; + } + + public PullParamBuilder setTimeoutMillis(long timeoutMillis) { + this.timeoutMillis = timeoutMillis; + return this; + } + + public PullParamBuilder setRequestTimeoutMillis(long requestTimeoutMillis) { + this.requestTimeoutMillis = requestTimeoutMillis; + return this; + } + + public PullParamBuilder setConsumeOffset(long consumeOffset) { + this.consumeOffset = consumeOffset; + return this; + } + + public PullParamBuilder setMinPullOffset(long minPullOffset) { + this.minPullOffset = minPullOffset; + return this; + } + + public PullParamBuilder setMaxPullOffset(long maxPullOffset) { + this.maxPullOffset = maxPullOffset; + return this; + } + } +} diff --git a/qmq-client/src/main/java/qunar/tc/qmq/consumer/pull/PullRegister.java b/qmq-client/src/main/java/qunar/tc/qmq/consumer/pull/PullRegister.java new file mode 100644 index 00000000..70b823a5 --- /dev/null +++ b/qmq-client/src/main/java/qunar/tc/qmq/consumer/pull/PullRegister.java @@ -0,0 +1,220 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.consumer.pull; + +import qunar.tc.qmq.broker.BrokerService; +import qunar.tc.qmq.broker.impl.BrokerServiceImpl; +import qunar.tc.qmq.common.MapKeyBuilder; +import qunar.tc.qmq.common.StatusSource; +import qunar.tc.qmq.concurrent.NamedThreadFactory; +import qunar.tc.qmq.consumer.exception.DuplicateListenerException; +import qunar.tc.qmq.consumer.register.ConsumerRegister; +import qunar.tc.qmq.consumer.register.RegistParam; +import qunar.tc.qmq.metainfoclient.ConsumerStateChangedListener; +import qunar.tc.qmq.metainfoclient.MetaInfoService; +import qunar.tc.qmq.utils.RetrySubjectUtils; + +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; + +import static qunar.tc.qmq.common.StatusSource.*; + +/** + * @author yiqun.fan create on 17-8-17. + */ +public class PullRegister implements ConsumerRegister, ConsumerStateChangedListener { + private volatile Boolean isOnline = false; + + private final Map pullEntryMap = new HashMap<>(); + + private final Map pullConsumerMap = new HashMap<>(); + + private final ExecutorService pullExecutor = Executors.newCachedThreadPool(new NamedThreadFactory("qmq-pull")); + + private final MetaInfoService metaInfoService; + private final BrokerService brokerService; + private final PullService pullService; + private final AckService ackService; + + private String clientId; + private String metaServer; + private int destroyWaitInSeconds; + + public PullRegister() { + this.metaInfoService = new MetaInfoService(); + this.brokerService = new BrokerServiceImpl(metaInfoService); + this.pullService = new PullService(); + this.ackService = new AckService(this.brokerService); + } + + public void init() { + this.metaInfoService.setMetaServer(metaServer); + this.metaInfoService.setClientId(clientId); + this.metaInfoService.init(); + + this.ackService.setDestroyWaitInSeconds(destroyWaitInSeconds); + this.ackService.setClientId(clientId); + this.metaInfoService.setConsumerStateChangedListener(this); + } + + @Override + public synchronized void regist(String subject, String group, RegistParam param) { + registPullEntry(subject, group, param, new AlwaysPullStrategy()); + registPullEntry(RetrySubjectUtils.buildRetrySubject(subject, group), group, param, new WeightPullStrategy()); + } + + private void registPullEntry(String subject, String group, RegistParam param, PullStrategy pullStrategy) { + final String subscribeKey = MapKeyBuilder.buildSubscribeKey(subject, group); + PullEntry pullEntry = pullEntryMap.get(subscribeKey); + if (pullEntry == PullEntry.EMPTY_PULL_ENTRY) { + throw new DuplicateListenerException(subscribeKey); + } + if (pullEntry == null) { + pullEntry = createAndSubmitPullEntry(subject, group, param, pullStrategy); + } + if (isOnline) { + pullEntry.online(param.getActionSrc()); + } else { + pullEntry.offline(param.getActionSrc()); + } + } + + private PullEntry createAndSubmitPullEntry(String subject, String group, RegistParam param, PullStrategy pullStrategy) { + PushConsumerImpl pushConsumer = new PushConsumerImpl(subject, group, param); + PullEntry pullEntry = new PullEntry(pushConsumer, pullService, ackService, brokerService, pullStrategy); + pullEntryMap.put(MapKeyBuilder.buildSubscribeKey(subject, group), pullEntry); + pullExecutor.submit(pullEntry); + return pullEntry; + } + + DefaultPullConsumer createDefaultPullConsumer(String subject, String group, boolean isBroadcast) { + DefaultPullConsumer pullConsumer = new DefaultPullConsumer(subject, group, isBroadcast, clientId, pullService, ackService, brokerService); + registerDefaultPullConsumer(pullConsumer); + return pullConsumer; + } + + private synchronized void registerDefaultPullConsumer(DefaultPullConsumer pullConsumer) { + final String subscribeKey = MapKeyBuilder.buildSubscribeKey(pullConsumer.subject(), pullConsumer.group()); + if (pullEntryMap.containsKey(subscribeKey)) { + throw new DuplicateListenerException(subscribeKey); + } + pullEntryMap.put(subscribeKey, PullEntry.EMPTY_PULL_ENTRY); + pullConsumerMap.put(subscribeKey, pullConsumer); + pullExecutor.submit(pullConsumer); + } + + @Override + public void unregist(String subject, String group) { + changeOnOffline(subject, group, false, CODE); + } + + @Override + public void online(String subject, String group) { + changeOnOffline(subject, group, true, OPS); + } + + @Override + public void offline(String subject, String group) { + changeOnOffline(subject, group, false, OPS); + } + + private synchronized void changeOnOffline(String subject, String group, boolean isOnline, StatusSource src) { + final String realSubject = RetrySubjectUtils.getRealSubject(subject); + final String retrySubject = RetrySubjectUtils.buildRetrySubject(realSubject, group); + + final String key = MapKeyBuilder.buildSubscribeKey(realSubject, group); + final PullEntry pullEntry = pullEntryMap.get(key); + changeOnOffline(pullEntry, isOnline, src); + + final PullEntry retryPullEntry = pullEntryMap.get(MapKeyBuilder.buildSubscribeKey(retrySubject, group)); + changeOnOffline(retryPullEntry, isOnline, src); + + final DefaultPullConsumer pullConsumer = pullConsumerMap.get(key); + if (pullConsumer == null) return; + + if (isOnline) { + pullConsumer.online(src); + } else { + pullConsumer.offline(src); + } + } + + private void changeOnOffline(PullEntry pullEntry, boolean isOnline, StatusSource src) { + if (pullEntry == null) return; + + if (isOnline) { + pullEntry.online(src); + } else { + pullEntry.offline(src); + } + } + + @Override + public synchronized void setAutoOnline(boolean autoOnline) { + if (autoOnline) { + online(); + } else { + offline(); + } + isOnline = autoOnline; + } + + public synchronized boolean offline() { + isOnline = false; + for (PullEntry pullEntry : pullEntryMap.values()) { + pullEntry.offline(HEALTHCHECKER); + } + for (DefaultPullConsumer pullConsumer : pullConsumerMap.values()) { + pullConsumer.offline(HEALTHCHECKER); + } + ackService.tryCleanAck(); + return true; + } + + public synchronized boolean online() { + isOnline = true; + for (PullEntry pullEntry : pullEntryMap.values()) { + pullEntry.online(HEALTHCHECKER); + } + for (DefaultPullConsumer pullConsumer : pullConsumerMap.values()) { + pullConsumer.online(HEALTHCHECKER); + } + return true; + } + + public void setClientId(String clientId) { + this.clientId = clientId; + } + + public void setMetaServer(String metaServer) { + this.metaServer = metaServer; + } + + @Override + public synchronized void destroy() { + for (PullEntry pullEntry : pullEntryMap.values()) { + pullEntry.destroy(); + } + ackService.destroy(); + } + + public void setDestroyWaitInSeconds(int destroyWaitInSeconds) { + this.destroyWaitInSeconds = destroyWaitInSeconds; + } +} diff --git a/qmq-client/src/main/java/qunar/tc/qmq/consumer/pull/PullResult.java b/qmq-client/src/main/java/qunar/tc/qmq/consumer/pull/PullResult.java new file mode 100644 index 00000000..58b265f0 --- /dev/null +++ b/qmq-client/src/main/java/qunar/tc/qmq/consumer/pull/PullResult.java @@ -0,0 +1,49 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.consumer.pull; + +import qunar.tc.qmq.base.BaseMessage; +import qunar.tc.qmq.broker.BrokerGroupInfo; + +import java.util.List; + +/** + * @author yiqun.fan create on 17-8-18. + */ +class PullResult { + private final short responseCode; + private final List messages; + private final BrokerGroupInfo brokerGroup; + + public PullResult(short responseCode, List messages, BrokerGroupInfo brokerGroup) { + this.responseCode = responseCode; + this.messages = messages; + this.brokerGroup = brokerGroup; + } + + public short getResponseCode() { + return responseCode; + } + + public List getMessages() { + return messages; + } + + public BrokerGroupInfo getBrokerGroup() { + return brokerGroup; + } +} diff --git a/qmq-client/src/main/java/qunar/tc/qmq/consumer/pull/PullService.java b/qmq-client/src/main/java/qunar/tc/qmq/consumer/pull/PullService.java new file mode 100644 index 00000000..a4e923c8 --- /dev/null +++ b/qmq-client/src/main/java/qunar/tc/qmq/consumer/pull/PullService.java @@ -0,0 +1,263 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.consumer.pull; + +import com.google.common.collect.Lists; +import com.google.common.util.concurrent.AbstractFuture; +import io.netty.buffer.ByteBuf; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import qunar.tc.qmq.base.BaseMessage; +import qunar.tc.qmq.broker.BrokerGroupInfo; +import qunar.tc.qmq.config.PullSubjectsConfig; +import qunar.tc.qmq.consumer.pull.exception.PullException; +import qunar.tc.qmq.metrics.Metrics; +import qunar.tc.qmq.netty.client.NettyClient; +import qunar.tc.qmq.netty.client.ResponseFuture; +import qunar.tc.qmq.protocol.CommandCode; +import qunar.tc.qmq.protocol.Datagram; +import qunar.tc.qmq.protocol.consumer.PullRequest; +import qunar.tc.qmq.protocol.consumer.PullRequestPayloadHolder; +import qunar.tc.qmq.util.RemotingBuilder; +import qunar.tc.qmq.utils.Flags; +import qunar.tc.qmq.utils.PayloadHolderUtils; +import qunar.tc.qmq.utils.RetrySubjectUtils; + +import java.util.*; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; + +import static qunar.tc.qmq.metrics.MetricsConstants.SUBJECT_ARRAY; +import static qunar.tc.qmq.metrics.MetricsConstants.SUBJECT_GROUP_ARRAY; + +/** + * @author yiqun.fan create on 17-8-17. + */ +class PullService { + private static final Logger LOGGER = LoggerFactory.getLogger(PullService.class); + + private final NettyClient client = NettyClient.getClient(); + + PullResult pull(final PullParam pullParam) throws ExecutionException, InterruptedException { + final PullResultFuture result = new PullResultFuture(pullParam.getBrokerGroup()); + pull(pullParam, result); + return result.get(); + } + + private void pull(final PullParam pullParam, final PullCallback callback) { + final PullRequest request = buildPullRequest(pullParam); + Datagram datagram = RemotingBuilder.buildRequestDatagram(CommandCode.PULL_MESSAGE, new PullRequestPayloadHolder(request)); + long networkTripTimeout = pullParam.getRequestTimeoutMillis(); + long pullProcessTimeout = pullParam.getTimeoutMillis(); + long responseTimeout = pullProcessTimeout < 0 ? networkTripTimeout : (networkTripTimeout + pullProcessTimeout); + + try { + client.sendAsync(pullParam.getBrokerGroup().getMaster(), datagram, responseTimeout, new PullCallbackWrapper(request, callback)); + } catch (Exception e) { + monitorPullError(pullParam.getSubject(), pullParam.getGroup()); + callback.onException(e); + } + } + + private PullRequest buildPullRequest(PullParam pullParam) { + PullRequest request = new PullRequest(); + request.setSubject(pullParam.getSubject()); + request.setGroup(pullParam.getGroup()); + request.setRequestNum(pullParam.getPullBatchSize()); + request.setTimeoutMillis(pullParam.getTimeoutMillis()); + request.setOffset(pullParam.getConsumeOffset()); + request.setPullOffsetBegin(pullParam.getMinPullOffset()); + request.setPullOffsetLast(pullParam.getMaxPullOffset()); + request.setConsumerId(pullParam.getConsumerId()); + request.setBroadcast(pullParam.isBroadcast()); + request.setTagTypeCode(pullParam.getTagType().getCode()); + List tags = new ArrayList<>(pullParam.getTags().size()); + for (String tag : pullParam.getTags()) { + tags.add(tag.getBytes()); + } + request.setTags(tags); + return request; + } + + public interface PullCallback { + void onCompleted(short responseCode, List messages); + + void onException(Exception ex); + } + + private static final class PullCallbackWrapper implements ResponseFuture.Callback { + private final PullRequest request; + private final PullCallback callback; + + PullCallbackWrapper(PullRequest request, PullCallback callback) { + this.request = request; + this.callback = callback; + } + + @Override + public void processResponse(ResponseFuture responseFuture) { + try { + doProcessResponse(responseFuture); + } catch (Exception e) { + monitorPullError(request.getSubject(), request.getGroup()); + callback.onException(e); + } + } + + private void doProcessResponse(ResponseFuture responseFuture) { + monitorPullTime(request.getSubject(), request.getGroup(), responseFuture.getRequestCostTime()); + if (!responseFuture.isSendOk()) { + monitorPullError(request.getSubject(), request.getGroup()); + callback.onException(new PullException("send fail. opaque=" + responseFuture.getOpaque())); + return; + } + + final Datagram response = responseFuture.getResponse(); + if (response == null) { + monitorPullError(request.getSubject(), request.getGroup()); + if (responseFuture.isTimeout()) { + callback.onException(new TimeoutException("pull message " + request.getSubject() + " timeout response: opaque=" + responseFuture.getOpaque() + ". " + responseFuture)); + } else { + callback.onException(new PullException("pull message " + request.getSubject() + "no receive response: opaque=" + responseFuture.getOpaque() + ". " + responseFuture)); + } + return; + } + + try { + handleResponse(response); + } finally { + response.release(); + } + } + + private void handleResponse(final Datagram response) { + final short responseCode = response.getHeader().getCode(); + if (responseCode == CommandCode.NO_MESSAGE) { + callback.onCompleted(responseCode, Collections.emptyList()); + } else if (responseCode != CommandCode.SUCCESS) { + monitorPullError(request.getSubject(), request.getGroup()); + callback.onCompleted(responseCode, Collections.emptyList()); + } else { + List messages = deserializeBaseMessage(response.getBody()); + if (messages == null) { + messages = Collections.emptyList(); + } + monitorPullCount(request.getSubject(), request.getGroup(), messages.size()); + for (BaseMessage message : messages) { + if (message.getMaxRetryNum() < 0) { + String realSubject = RetrySubjectUtils.getRealSubject(message.getSubject()); + message.setMaxRetryNum(PullSubjectsConfig.get().getMaxRetryNum(realSubject).get()); + } + } + callback.onCompleted(responseCode, messages); + } + } + + private List deserializeBaseMessage(ByteBuf input) { + if (input.readableBytes() == 0) return Collections.emptyList(); + List result = Lists.newArrayList(); + + long pullLogOffset = input.readLong(); + //ignore consumer offset + input.readLong(); + + while (input.isReadable()) { + BaseMessage message = new BaseMessage(); + byte flag = input.readByte(); + input.skipBytes(8 + 8); + String subject = PayloadHolderUtils.readString(input); + String messageId = PayloadHolderUtils.readString(input); + readTags(input, message, flag); + int bodyLen = input.readInt(); + ByteBuf body = input.readSlice(bodyLen); + HashMap attrs = deserializeMapWrapper(subject, messageId, body); + message.setMessageId(messageId); + message.setSubject(subject); + message.setAttrs(attrs); + message.setProperty(BaseMessage.keys.qmq_pullOffset, pullLogOffset); + result.add(message); + + pullLogOffset++; + } + return result; + } + + private void readTags(ByteBuf input, BaseMessage message, byte flag) { + if (!Flags.hasTags(flag)) return; + + final byte tagsSize = input.readByte(); + for (int i = 0; i < tagsSize; i++) { + final String tag = PayloadHolderUtils.readString(input); + message.addTag(tag); + } + } + + private HashMap deserializeMapWrapper(String subject, String messageId, ByteBuf body) { + try { + return deserializeMap(body); + } catch (Exception e) { + LOGGER.error("deserialize message failed subject:{} messageId: {}", subject, messageId); + Metrics.counter("qmq_pull_deserialize_fail_count", SUBJECT_ARRAY, new String[]{subject}).inc(); + HashMap result = new HashMap<>(); + result.put(BaseMessage.keys.qmq_corruptData.name(), "true"); + result.put(BaseMessage.keys.qmq_createTime.name(), new Date().getTime()); + return result; + } + } + + private HashMap deserializeMap(ByteBuf body) { + HashMap map = new HashMap<>(); + while (body.isReadable(4)) { + String key = PayloadHolderUtils.readString(body); + String value = PayloadHolderUtils.readString(body); + map.put(key, value); + } + return map; + } + } + + private static final class PullResultFuture extends AbstractFuture implements PullCallback { + private final BrokerGroupInfo brokerGroup; + + PullResultFuture(BrokerGroupInfo brokerGroup) { + this.brokerGroup = brokerGroup; + } + + @Override + public void onCompleted(short responseCode, List messages) { + super.set(new PullResult(responseCode, messages, brokerGroup)); + } + + @Override + public void onException(Exception ex) { + super.setException(ex); + } + } + + private static void monitorPullTime(String subject, String group, long time) { + Metrics.timer("qmq_pull_timer", SUBJECT_GROUP_ARRAY, new String[]{subject, group}).update(time, TimeUnit.MILLISECONDS); + } + + private static void monitorPullError(String subject, String group) { + Metrics.counter("qmq_pull_error", SUBJECT_GROUP_ARRAY, new String[]{subject, group}).inc(); + } + + private static void monitorPullCount(String subject, String group, int pullSize) { + Metrics.counter("qmq_pull_count", SUBJECT_GROUP_ARRAY, new String[]{subject, group}).inc(pullSize); + } +} diff --git a/qmq-client/src/main/java/qunar/tc/qmq/consumer/pull/PullStrategy.java b/qmq-client/src/main/java/qunar/tc/qmq/consumer/pull/PullStrategy.java new file mode 100644 index 00000000..e456c5a4 --- /dev/null +++ b/qmq-client/src/main/java/qunar/tc/qmq/consumer/pull/PullStrategy.java @@ -0,0 +1,23 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.consumer.pull; + +interface PullStrategy { + boolean needPull(); + + void record(boolean status); +} diff --git a/qmq-client/src/main/java/qunar/tc/qmq/consumer/pull/PulledMessage.java b/qmq-client/src/main/java/qunar/tc/qmq/consumer/pull/PulledMessage.java new file mode 100644 index 00000000..46d98dac --- /dev/null +++ b/qmq-client/src/main/java/qunar/tc/qmq/consumer/pull/PulledMessage.java @@ -0,0 +1,57 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.consumer.pull; + +import qunar.tc.qmq.base.BaseMessage; +import qunar.tc.qmq.consumer.ConsumeMessage; + +import java.util.concurrent.atomic.AtomicBoolean; + +/** + * @author yiqun.fan create on 17-7-20. + */ +class PulledMessage extends ConsumeMessage { + private transient final AckEntry ackEntry; + private transient final AckHook ackHook; + private transient final AtomicBoolean hasAcked = new AtomicBoolean(false); + + PulledMessage(BaseMessage message, AckEntry ackEntry, AckHook ackHook) { + super(message); + this.ackEntry = ackEntry; + this.ackHook = ackHook; + } + + AckEntry ackEntry() { + return ackEntry; + } + + boolean hasNotAcked() { + return !hasAcked.get(); + } + + @Override + public void ack(long elapsed, Throwable e) { + if (!hasAcked.compareAndSet(false, true)) { + return; + } + if (ackHook != null) { + ackHook.call(this, e); + } else { + AckHelper.ack(this, e); + } + } +} diff --git a/qmq-client/src/main/java/qunar/tc/qmq/consumer/pull/PulledMessageFilter.java b/qmq-client/src/main/java/qunar/tc/qmq/consumer/pull/PulledMessageFilter.java new file mode 100644 index 00000000..4886991e --- /dev/null +++ b/qmq-client/src/main/java/qunar/tc/qmq/consumer/pull/PulledMessageFilter.java @@ -0,0 +1,28 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.consumer.pull; + +/** + * @author yiqun.fan create on 17-11-2. + */ +interface PulledMessageFilter { + + /** + * @return true时,保留;false时,丢弃 + */ + boolean filter(final PulledMessage message); +} diff --git a/qmq-client/src/main/java/qunar/tc/qmq/consumer/pull/PushConsumer.java b/qmq-client/src/main/java/qunar/tc/qmq/consumer/pull/PushConsumer.java new file mode 100644 index 00000000..8e88ed15 --- /dev/null +++ b/qmq-client/src/main/java/qunar/tc/qmq/consumer/pull/PushConsumer.java @@ -0,0 +1,33 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.consumer.pull; + +import qunar.tc.qmq.BaseConsumer; + +import java.util.List; + +/** + * @author yiqun.fan create on 17-9-12. + */ +interface PushConsumer extends BaseConsumer, AckHook { + + ConsumeParam consumeParam(); + + void push(List messages); + + boolean cleanLocalBuffer(); +} diff --git a/qmq-client/src/main/java/qunar/tc/qmq/consumer/pull/PushConsumerImpl.java b/qmq-client/src/main/java/qunar/tc/qmq/consumer/pull/PushConsumerImpl.java new file mode 100644 index 00000000..e8ea2b30 --- /dev/null +++ b/qmq-client/src/main/java/qunar/tc/qmq/consumer/pull/PushConsumerImpl.java @@ -0,0 +1,153 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.consumer.pull; + +import com.google.common.base.Supplier; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import qunar.tc.qmq.base.BaseMessage; +import qunar.tc.qmq.consumer.BaseMessageHandler; +import qunar.tc.qmq.consumer.register.RegistParam; +import qunar.tc.qmq.metrics.Metrics; +import qunar.tc.qmq.metrics.QmqCounter; +import qunar.tc.qmq.metrics.QmqTimer; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.RejectedExecutionException; +import java.util.concurrent.TimeUnit; + +import static qunar.tc.qmq.metrics.MetricsConstants.SUBJECT_GROUP_ARRAY; + +/** + * @author yiqun.fan create on 17-8-18. + */ +class PushConsumerImpl extends BaseMessageHandler implements PushConsumer { + private static final Logger LOGGER = LoggerFactory.getLogger(PushConsumer.class); + + private final ConsumeParam consumeParam; + private final LinkedBlockingQueue messageBuffer = new LinkedBlockingQueue<>(); + + private final QmqTimer createToHandleTimer; + private final QmqTimer handleTimer; + private final QmqCounter handleFailCounter; + + PushConsumerImpl(String subject, String group, RegistParam param) { + super(param.getExecutor(), param.getMessageListener()); + this.consumeParam = new ConsumeParam(subject, group, param); + + String[] values = {subject, group}; + Metrics.gauge("qmq_pull_buffer_size", SUBJECT_GROUP_ARRAY, values, new Supplier() { + @Override + public Double get() { + return (double) messageBuffer.size(); + } + }); + this.createToHandleTimer = Metrics.timer("qmq_pull_createToHandle_timer", SUBJECT_GROUP_ARRAY, values); + this.handleTimer = Metrics.timer("qmq_pull_handle_timer", SUBJECT_GROUP_ARRAY, values); + this.handleFailCounter = Metrics.counter("qmq_pull_handleFail_count", SUBJECT_GROUP_ARRAY, values); + } + + @Override + public String subject() { + return consumeParam.getSubject(); + } + + @Override + public String group() { + return consumeParam.getGroup(); + } + + @Override + public ConsumeParam consumeParam() { + return consumeParam; + } + + @Override + public boolean cleanLocalBuffer() { + while (!messageBuffer.isEmpty()) { + if (!push(messageBuffer.peek())) { + return false; + } else { + messageBuffer.poll(); + } + } + return true; + } + + @Override + public void push(List messages) { + for (int i = 0; i < messages.size(); i++) { + final PulledMessage message = messages.get(i); + if (!push(message)) { + messageBuffer.addAll(messages.subList(i, messages.size())); + break; + } + } + } + + private boolean push(PulledMessage message) { + HandleTaskImpl task = new HandleTaskImpl(message, this); + try { + executor.execute(task); + LOGGER.info("进入执行队列 {}:{}", message.getSubject(), message.getMessageId()); + return true; + } catch (RejectedExecutionException e) { + LOGGER.error("消息进入执行队列失败,请调整消息处理线程池大小, {}:{}", message.getSubject(), message.getMessageId()); + return false; + } + } + + @Override + protected void ack(BaseMessage message, long elapsed, Throwable exception, Map attachment) { + PulledMessage pulledMessage = (PulledMessage) message; + if (pulledMessage.hasNotAcked()) { + AckHelper.ackWithTrace(pulledMessage, exception); + } + } + + private final class HandleTaskImpl extends HandleTask { + private final PulledMessage message; + + HandleTaskImpl(PulledMessage message, BaseMessageHandler handler) { + super(message, handler); + this.message = message; + } + + @Override + public void run() { + long start = System.currentTimeMillis(); + createToHandleTimer.update(start - message.getCreatedTime().getTime(), TimeUnit.MILLISECONDS); + try { + super.run(); + } finally { + handleTimer.update(System.currentTimeMillis() - start, TimeUnit.MILLISECONDS); + if (super.handleFail) { + handleFailCounter.inc(); + } + } + } + } + + @Override + public void call(PulledMessage message, Throwable throwable) { + applyPostOnMessage(message, throwable, new HashMap<>(message.filterContext())); + AckHelper.ackWithTrace(message, throwable); + } +} diff --git a/qmq-client/src/main/java/qunar/tc/qmq/consumer/pull/SendMessageBack.java b/qmq-client/src/main/java/qunar/tc/qmq/consumer/pull/SendMessageBack.java new file mode 100644 index 00000000..0e70b310 --- /dev/null +++ b/qmq-client/src/main/java/qunar/tc/qmq/consumer/pull/SendMessageBack.java @@ -0,0 +1,36 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.consumer.pull; + +import qunar.tc.qmq.base.BaseMessage; +import qunar.tc.qmq.broker.BrokerGroupInfo; +import qunar.tc.qmq.common.ClientType; + +/** + * @author yiqun.fan create on 17-8-23. + */ +public interface SendMessageBack { + void sendBack(BrokerGroupInfo brokerGroup, BaseMessage messages, Callback callback, ClientType clientType); + + void sendBackAndCompleteNack(int nextRetryCount, BaseMessage message, AckEntry ackEntry); + + interface Callback { + void success(); + + void fail(Throwable e); + } +} diff --git a/qmq-client/src/main/java/qunar/tc/qmq/consumer/pull/SendMessageBackImpl.java b/qmq-client/src/main/java/qunar/tc/qmq/consumer/pull/SendMessageBackImpl.java new file mode 100644 index 00000000..1cbecd3e --- /dev/null +++ b/qmq-client/src/main/java/qunar/tc/qmq/consumer/pull/SendMessageBackImpl.java @@ -0,0 +1,197 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.consumer.pull; + +import com.google.common.collect.Lists; +import io.netty.util.Timeout; +import io.netty.util.TimerTask; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import qunar.tc.qmq.base.BaseMessage; +import qunar.tc.qmq.broker.BrokerClusterInfo; +import qunar.tc.qmq.broker.BrokerGroupInfo; +import qunar.tc.qmq.broker.BrokerLoadBalance; +import qunar.tc.qmq.broker.BrokerService; +import qunar.tc.qmq.broker.impl.PollBrokerLoadBalance; +import qunar.tc.qmq.common.ClientType; +import qunar.tc.qmq.common.TimerUtil; +import qunar.tc.qmq.consumer.pull.exception.SendMessageBackException; +import qunar.tc.qmq.metrics.Metrics; +import qunar.tc.qmq.netty.client.NettyClient; +import qunar.tc.qmq.netty.client.ResponseFuture; +import qunar.tc.qmq.netty.exception.ClientSendException; +import qunar.tc.qmq.protocol.CommandCode; +import qunar.tc.qmq.protocol.Datagram; +import qunar.tc.qmq.protocol.MessagesPayloadHolder; +import qunar.tc.qmq.protocol.QMQSerializer; +import qunar.tc.qmq.protocol.producer.MessageProducerCode; +import qunar.tc.qmq.protocol.producer.SendResult; +import qunar.tc.qmq.util.RemotingBuilder; + +import java.util.Date; +import java.util.Map; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; + +import static qunar.tc.qmq.protocol.CommandCode.SEND_MESSAGE; + +/** + * @author yiqun.fan create on 17-8-23. + */ +class SendMessageBackImpl implements SendMessageBack { + private static final Logger LOGGER = LoggerFactory.getLogger(SendMessageBackImpl.class); + private static final long SEND_MESSAGE_TIMEOUT = 5000; + private static final int SEND_BACK_DELAY_SECONDS = 5; + + private static final NettyClient NETTY_CLIENT = NettyClient.getClient(); + + private final BrokerService brokerService; + + private final BrokerLoadBalance brokerLoadBalance; + + SendMessageBackImpl(BrokerService brokerService) { + this.brokerService = brokerService; + this.brokerLoadBalance = PollBrokerLoadBalance.getInstance(); + } + + @Override + public void sendBack(final BrokerGroupInfo brokerGroup, BaseMessage message, final Callback callback, final ClientType clientType) { + if (message == null) { + if (callback != null) { + callback.success(); + } + return; + } + message.setProperty(BaseMessage.keys.qmq_createTime, new Date().getTime()); + final Datagram datagram = RemotingBuilder.buildRequestDatagram(SEND_MESSAGE, new MessagesPayloadHolder(Lists.newArrayList(message))); + + final String subject = message.getSubject(); + try { + NETTY_CLIENT.sendAsync(brokerGroup.getMaster(), datagram, SEND_MESSAGE_TIMEOUT, + new ResponseFuture.Callback() { + @Override + public void processResponse(ResponseFuture responseFuture) { + final Datagram response = responseFuture.getResponse(); + if (!responseFuture.isSendOk() || response == null) { + monitorSendError(subject, -1); + callback.fail(new SendMessageBackException("send fail")); + } else { + try { + handleResponse(response); + } finally { + response.release(); + } + } + } + + private void handleResponse(final Datagram response) { + final int respCode = response.getHeader().getCode(); + final SendResult sendResult = getSendResult(response); + if (sendResult == null) { + monitorSendError(subject, 100 + respCode); + callback.fail(new SendMessageBackException("responseCode=" + respCode)); + return; + } + + if (respCode == CommandCode.SUCCESS && sendResult.getCode() == MessageProducerCode.SUCCESS) { + callback.success(); + return; + } + + if (respCode == CommandCode.BROKER_REJECT || sendResult.getCode() == MessageProducerCode.BROKER_READ_ONLY) { + brokerGroup.setAvailable(false); + brokerService.refresh(clientType, subject); + } + + monitorSendError(subject, 100 + respCode); + callback.fail(new SendMessageBackException("responseCode=" + respCode + ", sendCode=" + sendResult.getCode())); + } + }); + } catch (ClientSendException e) { + LOGGER.error("send message error. subject={}", subject); + monitorSendError(subject, e.getSendErrorCode().ordinal()); + callback.fail(e); + } catch (Exception e) { + LOGGER.error("send message error. subject={}", subject); + monitorSendError(subject, -1); + callback.fail(e); + } + } + + private SendResult getSendResult(Datagram response) { + try { + Map result = QMQSerializer.deserializeSendResultMap(response.getBody()); + if (result.isEmpty()) { + return SendResult.OK; + } + return result.values().iterator().next(); + } catch (Exception e) { + LOGGER.error("sendback exception on deserializeSendResultMap.", e); + return null; + } + } + + public void sendBackAndCompleteNack(final int nextRetryCount, final BaseMessage message, final AckEntry ackEntry) { + final BrokerClusterInfo brokerCluster = brokerService.getClusterBySubject(ClientType.PRODUCER, message.getSubject()); + final SendMessageBack.Callback callback = new SendMessageBack.Callback() { + private final int retryTooMuch = brokerCluster.getGroups().size() * 2; + private final AtomicInteger retryNumOnFail = new AtomicInteger(0); + + @Override + public void success() { + ackEntry.completed(); + } + + @Override + public void fail(Throwable e) { + if (retryNumOnFail.incrementAndGet() > retryTooMuch) { + if (e instanceof SendMessageBackException) { + LOGGER.error("send message back fail, and retry {} times after {} seconds. exception: {}", retryNumOnFail.get(), SEND_BACK_DELAY_SECONDS, e.getMessage()); + } else { + LOGGER.error("send message back fail, and retry {} times after {} seconds", retryNumOnFail.get(), SEND_BACK_DELAY_SECONDS, e); + } + final SendMessageBack.Callback callback1 = this; + TimerUtil.newTimeout(new TimerTask() { + @Override + public void run(Timeout timeout) { + SendMessageBackImpl.this.sendBackAndCompleteNack(message, callback1); + } + }, SEND_BACK_DELAY_SECONDS, TimeUnit.SECONDS); + } else { + if (e instanceof SendMessageBackException) { + LOGGER.error("send message back fail, and retry {} times. exception: {}", retryNumOnFail.get(), SEND_BACK_DELAY_SECONDS, e.getMessage()); + } else { + LOGGER.error("send message back fail, and retry {} times", retryNumOnFail.get(), SEND_BACK_DELAY_SECONDS, e); + } + SendMessageBackImpl.this.sendBackAndCompleteNack(message, this); + } + } + }; + final BrokerGroupInfo brokerGroup = brokerLoadBalance.loadBalance(brokerCluster, null); + sendBack(brokerGroup, message, callback, ClientType.PRODUCER); + } + + private void sendBackAndCompleteNack(final BaseMessage message, final SendMessageBack.Callback callback) { + final BrokerClusterInfo brokerCluster = brokerService.getClusterBySubject(ClientType.PRODUCER, message.getSubject()); + final BrokerGroupInfo brokerGroup = brokerLoadBalance.loadBalance(brokerCluster, null); + sendBack(brokerGroup, message, callback, ClientType.PRODUCER); + } + + private static void monitorSendError(String subject, int errorCode) { + Metrics.counter("qmq_pull_send_msg_error", new String[]{"subject", "error"}, new String[]{subject, String.valueOf(errorCode)}).inc(); + } +} diff --git a/qmq-client/src/main/java/qunar/tc/qmq/consumer/pull/WeightLoadBalance.java b/qmq-client/src/main/java/qunar/tc/qmq/consumer/pull/WeightLoadBalance.java new file mode 100644 index 00000000..e5635d5e --- /dev/null +++ b/qmq-client/src/main/java/qunar/tc/qmq/consumer/pull/WeightLoadBalance.java @@ -0,0 +1,110 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.consumer.pull; + +import qunar.tc.qmq.broker.BrokerClusterInfo; +import qunar.tc.qmq.broker.BrokerGroupInfo; + +import java.util.List; +import java.util.Random; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; + +/** + * Created by zhaohui.yu + * 5/29/18 + */ +class WeightLoadBalance { + + private static final int MIN_WEIGHT = 5; + + private static final int MAX_WEIGHT = 150; + + private static final int DEFAULT_WEIGHT = 100; + + private final Random random = new Random(); + + private ConcurrentMap weights = new ConcurrentHashMap<>(); + + BrokerGroupInfo select(BrokerClusterInfo cluster) { + List groups = cluster.getGroups(); + if (groups == null || groups.isEmpty()) return null; + + int size = groups.size(); + int totalWeight = 0; + boolean sameWeight = true; + int lastWeight = -1; + for (int i = 0; i < size; ++i) { + BrokerGroupInfo group = groups.get(i); + if (!group.isAvailable()) continue; + Integer weight = weights.get(group); + if (weight == null) { + weights.putIfAbsent(group, DEFAULT_WEIGHT); + weight = weights.get(group); + } + if (lastWeight != -1 && lastWeight != weight) { + sameWeight = false; + } + lastWeight = weight; + totalWeight += weight; + } + if (totalWeight == 0) return null; + + if (totalWeight > 0 && !sameWeight) { + int offset = random.nextInt(totalWeight); + for (int i = 0; i < size; ++i) { + BrokerGroupInfo group = groups.get(i); + if (!group.isAvailable()) continue; + Integer weight = weights.get(group); + offset -= weight; + if (offset <= 0) { + return group; + } + } + } + + int index = random.nextInt(size); + return groups.get(index); + } + + public void timeout(BrokerGroupInfo group) { + update(group, 0.25, DEFAULT_WEIGHT); + } + + void noMessage(BrokerGroupInfo group) { + update(group, 0.75, DEFAULT_WEIGHT); + } + + void fetchedMessages(BrokerGroupInfo group) { + update(group, 1.25, DEFAULT_WEIGHT); + } + + void fetchedEnoughMessages(BrokerGroupInfo group) { + update(group, 1.5, MAX_WEIGHT); + } + + private void update(BrokerGroupInfo group, double factor, int maxWeight) { + Integer weight = weights.get(group); + if (weight == null) { + weights.putIfAbsent(group, DEFAULT_WEIGHT); + weight = weights.get(group); + } + + weight = Math.min(Math.max((int) (weight * factor), MIN_WEIGHT), maxWeight); + weights.put(group, weight); + } +} diff --git a/qmq-client/src/main/java/qunar/tc/qmq/consumer/pull/WeightPullStrategy.java b/qmq-client/src/main/java/qunar/tc/qmq/consumer/pull/WeightPullStrategy.java new file mode 100644 index 00000000..81a52ef1 --- /dev/null +++ b/qmq-client/src/main/java/qunar/tc/qmq/consumer/pull/WeightPullStrategy.java @@ -0,0 +1,44 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.consumer.pull; + +import java.util.concurrent.ThreadLocalRandom; + +class WeightPullStrategy implements PullStrategy { + private static final int MIN_WEIGHT = 1; + private static final int MAX_WEIGHT = 32; + + private int currentWeight = MAX_WEIGHT; + + public boolean needPull() { + return randomWeightThreshold() < currentWeight; + } + + private int randomWeightThreshold() { + return ThreadLocalRandom.current().nextInt(0, MAX_WEIGHT); + } + + public void record(boolean status) { + if (status) { + int weight = currentWeight * 2; + currentWeight = Math.min(weight, MAX_WEIGHT); + } else { + int weight = currentWeight / 2; + currentWeight = Math.max(weight, MIN_WEIGHT); + } + } +} diff --git a/qmq-client/src/main/java/qunar/tc/qmq/consumer/pull/exception/AckException.java b/qmq-client/src/main/java/qunar/tc/qmq/consumer/pull/exception/AckException.java new file mode 100644 index 00000000..f46f583c --- /dev/null +++ b/qmq-client/src/main/java/qunar/tc/qmq/consumer/pull/exception/AckException.java @@ -0,0 +1,38 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.consumer.pull.exception; + +/** + * @author yiqun.fan create on 17-8-18. + */ +public class AckException extends Exception { + public AckException() { + super(); + } + + public AckException(String s) { + super(s); + } + + public AckException(String message, Throwable cause) { + super(message, cause); + } + + public AckException(Throwable cause) { + super(cause); + } +} diff --git a/qmq-client/src/main/java/qunar/tc/qmq/consumer/pull/exception/PullException.java b/qmq-client/src/main/java/qunar/tc/qmq/consumer/pull/exception/PullException.java new file mode 100644 index 00000000..134bd95c --- /dev/null +++ b/qmq-client/src/main/java/qunar/tc/qmq/consumer/pull/exception/PullException.java @@ -0,0 +1,38 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.consumer.pull.exception; + +/** + * @author yiqun.fan create on 17-8-18. + */ +public class PullException extends Exception { + public PullException() { + super(); + } + + public PullException(String s) { + super(s); + } + + public PullException(String message, Throwable cause) { + super(message, cause); + } + + public PullException(Throwable cause) { + super(cause); + } +} diff --git a/qmq-client/src/main/java/qunar/tc/qmq/consumer/pull/exception/SendMessageBackException.java b/qmq-client/src/main/java/qunar/tc/qmq/consumer/pull/exception/SendMessageBackException.java new file mode 100644 index 00000000..41bcfc7c --- /dev/null +++ b/qmq-client/src/main/java/qunar/tc/qmq/consumer/pull/exception/SendMessageBackException.java @@ -0,0 +1,38 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.consumer.pull.exception; + +/** + * @author yiqun.fan create on 17-8-18. + */ +public class SendMessageBackException extends Exception { + public SendMessageBackException() { + super(); + } + + public SendMessageBackException(String s) { + super(s); + } + + public SendMessageBackException(String message, Throwable cause) { + super(message, cause); + } + + public SendMessageBackException(Throwable cause) { + super(cause); + } +} diff --git a/qmq-client/src/main/java/qunar/tc/qmq/consumer/register/ConsumerRegister.java b/qmq-client/src/main/java/qunar/tc/qmq/consumer/register/ConsumerRegister.java new file mode 100644 index 00000000..42499faf --- /dev/null +++ b/qmq-client/src/main/java/qunar/tc/qmq/consumer/register/ConsumerRegister.java @@ -0,0 +1,33 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.consumer.register; + +/** + * User: zhaohuiyu + * Date: 6/5/13 + * Time: 10:59 AM + */ +public interface ConsumerRegister { + + void regist(String prefix, String group, RegistParam param); + + void unregist(String prefix, String group); + + void setAutoOnline(boolean autoOnline); + + void destroy(); +} diff --git a/qmq-client/src/main/java/qunar/tc/qmq/consumer/register/RegistParam.java b/qmq-client/src/main/java/qunar/tc/qmq/consumer/register/RegistParam.java new file mode 100644 index 00000000..0bfaa131 --- /dev/null +++ b/qmq-client/src/main/java/qunar/tc/qmq/consumer/register/RegistParam.java @@ -0,0 +1,75 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.consumer.register; + +import qunar.tc.qmq.MessageListener; +import qunar.tc.qmq.SubscribeParam; +import qunar.tc.qmq.common.StatusSource; + +import java.util.concurrent.Executor; + +/** + * @author yiqun.fan create on 17-8-23. + */ +public class RegistParam { + private final Executor executor; + private final MessageListener messageListener; + private final SubscribeParam subscribeParam; + private boolean isBroadcast = false; + private final String clientId; + + private volatile StatusSource actionSrc = StatusSource.HEALTHCHECKER; + + public RegistParam(Executor executor, MessageListener messageListener, SubscribeParam subscribeParam, String clientId) { + this.executor = executor; + this.messageListener = messageListener; + this.subscribeParam = subscribeParam; + this.clientId = clientId; + } + + public Executor getExecutor() { + return executor; + } + + public MessageListener getMessageListener() { + return messageListener; + } + + public SubscribeParam getSubscribeParam() { + return subscribeParam; + } + + public String getClientId() { + return this.clientId; + } + + public boolean isBroadcast() { + return isBroadcast; + } + + public void setBroadcast(boolean broadcast) { + isBroadcast = broadcast; + } + + public void setActionSrc(StatusSource actionSrc) { + this.actionSrc = actionSrc; + } + + public StatusSource getActionSrc() { + return this.actionSrc; + } +} diff --git a/qmq-client/src/main/java/qunar/tc/qmq/metainfoclient/ConsumerStateChangedListener.java b/qmq-client/src/main/java/qunar/tc/qmq/metainfoclient/ConsumerStateChangedListener.java new file mode 100644 index 00000000..dc1f2370 --- /dev/null +++ b/qmq-client/src/main/java/qunar/tc/qmq/metainfoclient/ConsumerStateChangedListener.java @@ -0,0 +1,27 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.metainfoclient; + +/** + * Created by zhaohui.yu + * 4/2/18 + */ +public interface ConsumerStateChangedListener { + void online(String subject, String consumerGroup); + + void offline(String subject, String consumerGroup); +} diff --git a/qmq-client/src/main/java/qunar/tc/qmq/metainfoclient/MetaInfo.java b/qmq-client/src/main/java/qunar/tc/qmq/metainfoclient/MetaInfo.java new file mode 100644 index 00000000..6b7a5151 --- /dev/null +++ b/qmq-client/src/main/java/qunar/tc/qmq/metainfoclient/MetaInfo.java @@ -0,0 +1,56 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.metainfoclient; + +import qunar.tc.qmq.broker.BrokerClusterInfo; +import qunar.tc.qmq.common.ClientType; + +/** + * @author yiqun.fan create on 17-8-31. + */ +public class MetaInfo { + private final String subject; + private final ClientType clientType; + private final BrokerClusterInfo clusterInfo; + + MetaInfo(String subject, ClientType clientType, BrokerClusterInfo clusterInfo) { + this.subject = subject; + this.clientType = clientType; + this.clusterInfo = clusterInfo; + } + + public String getSubject() { + return subject; + } + + public ClientType getClientType() { + return clientType; + } + + public BrokerClusterInfo getClusterInfo() { + return clusterInfo; + } + + @Override + public String toString() { + return "MetaInfo{" + + "subject='" + subject + '\'' + + ", clientType=" + clientType + + ", groups=" + clusterInfo.getGroups() + + '}'; + } +} diff --git a/qmq-client/src/main/java/qunar/tc/qmq/metainfoclient/MetaInfoClient.java b/qmq-client/src/main/java/qunar/tc/qmq/metainfoclient/MetaInfoClient.java new file mode 100644 index 00000000..dab090b6 --- /dev/null +++ b/qmq-client/src/main/java/qunar/tc/qmq/metainfoclient/MetaInfoClient.java @@ -0,0 +1,36 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.metainfoclient; + +import qunar.tc.qmq.meta.MetaServerLocator; +import qunar.tc.qmq.protocol.consumer.MetaInfoRequest; +import qunar.tc.qmq.protocol.consumer.MetaInfoResponse; + +/** + * @author yiqun.fan create on 17-9-1. + */ +interface MetaInfoClient { + void sendRequest(MetaInfoRequest request); + + void registerResponseSubscriber(ResponseSubscriber receiver); + + void setMetaServerLocator(MetaServerLocator locator); + + interface ResponseSubscriber { + void onResponse(MetaInfoResponse response); + } +} diff --git a/qmq-client/src/main/java/qunar/tc/qmq/metainfoclient/MetaInfoClientHandler.java b/qmq-client/src/main/java/qunar/tc/qmq/metainfoclient/MetaInfoClientHandler.java new file mode 100644 index 00000000..5ffd1625 --- /dev/null +++ b/qmq-client/src/main/java/qunar/tc/qmq/metainfoclient/MetaInfoClientHandler.java @@ -0,0 +1,106 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.metainfoclient; + +import io.netty.buffer.ByteBuf; +import io.netty.channel.ChannelHandler; +import io.netty.channel.ChannelHandlerContext; +import io.netty.channel.SimpleChannelInboundHandler; +import io.netty.util.internal.ConcurrentSet; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import qunar.tc.qmq.meta.BrokerCluster; +import qunar.tc.qmq.meta.BrokerGroup; +import qunar.tc.qmq.base.OnOfflineState; +import qunar.tc.qmq.meta.BrokerState; +import qunar.tc.qmq.protocol.CommandCode; +import qunar.tc.qmq.protocol.Datagram; +import qunar.tc.qmq.protocol.consumer.MetaInfoResponse; +import qunar.tc.qmq.utils.PayloadHolderUtils; + +import java.util.ArrayList; +import java.util.List; + +/** + * @author yiqun.fan create on 17-8-31. + */ +@ChannelHandler.Sharable +class MetaInfoClientHandler extends SimpleChannelInboundHandler { + private static final Logger LOG = LoggerFactory.getLogger(MetaInfoClientHandler.class); + + private final ConcurrentSet responseSubscribers = new ConcurrentSet<>(); + + void registerResponseSubscriber(MetaInfoClient.ResponseSubscriber subscriber) { + responseSubscribers.add(subscriber); + } + + @Override + protected void channelRead0(ChannelHandlerContext ctx, Datagram msg) { + MetaInfoResponse response = null; + if (msg.getHeader().getCode() == CommandCode.SUCCESS) { + response = deserializeMetaInfoResponse(msg.getBody()); + } + + if (response != null) { + notifySubscriber(response); + } else { + LOG.warn("request meta info UNKNOWN. code={}", msg.getHeader().getCode()); + } + } + + private void notifySubscriber(MetaInfoResponse response) { + for (MetaInfoClient.ResponseSubscriber subscriber : responseSubscribers) { + try { + subscriber.onResponse(response); + } catch (Exception e) { + LOG.error("", e); + } + } + } + + private static MetaInfoResponse deserializeMetaInfoResponse(ByteBuf buf) { + try { + final MetaInfoResponse metaInfoResponse = new MetaInfoResponse(); + metaInfoResponse.setTimestamp(buf.readLong()); + metaInfoResponse.setSubject(PayloadHolderUtils.readString(buf)); + metaInfoResponse.setConsumerGroup(PayloadHolderUtils.readString(buf)); + metaInfoResponse.setOnOfflineState(OnOfflineState.fromCode(buf.readByte())); + metaInfoResponse.setClientTypeCode(buf.readByte()); + metaInfoResponse.setBrokerCluster(deserializeBrokerCluster(buf)); + return metaInfoResponse; + } catch (Exception e) { + LOG.error("deserializeMetaInfoResponse exception", e); + } + return null; + } + + private static BrokerCluster deserializeBrokerCluster(ByteBuf buf) { + final int brokerGroupSize = buf.readShort(); + final List brokerGroups = new ArrayList<>(brokerGroupSize); + for (int i = 0; i < brokerGroupSize; i++) { + final BrokerGroup brokerGroup = new BrokerGroup(); + brokerGroup.setGroupName(PayloadHolderUtils.readString(buf)); + brokerGroup.setMaster(PayloadHolderUtils.readString(buf)); + brokerGroup.setUpdateTime(buf.readLong()); + final int brokerStateCode = buf.readByte(); + final BrokerState brokerState = BrokerState.codeOf(brokerStateCode); + brokerGroup.setBrokerState(brokerState); + brokerGroups.add(brokerGroup); + } + return new BrokerCluster(brokerGroups); + } +} diff --git a/qmq-client/src/main/java/qunar/tc/qmq/metainfoclient/MetaInfoClientNettyImpl.java b/qmq-client/src/main/java/qunar/tc/qmq/metainfoclient/MetaInfoClientNettyImpl.java new file mode 100644 index 00000000..842b0d38 --- /dev/null +++ b/qmq-client/src/main/java/qunar/tc/qmq/metainfoclient/MetaInfoClientNettyImpl.java @@ -0,0 +1,153 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.metainfoclient; + +import com.google.common.base.Optional; +import com.google.common.base.Supplier; +import com.google.common.base.Suppliers; +import io.netty.channel.Channel; +import io.netty.channel.ChannelFuture; +import io.netty.channel.ChannelFutureListener; +import io.netty.channel.ChannelInitializer; +import io.netty.channel.socket.SocketChannel; +import io.netty.util.concurrent.DefaultEventExecutorGroup; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import qunar.tc.qmq.meta.MetaServerLocator; +import qunar.tc.qmq.netty.DecodeHandler; +import qunar.tc.qmq.netty.EncodeHandler; +import qunar.tc.qmq.netty.NettyClientConfig; +import qunar.tc.qmq.netty.client.AbstractNettyClient; +import qunar.tc.qmq.netty.client.NettyConnectManageHandler; +import qunar.tc.qmq.protocol.CommandCode; +import qunar.tc.qmq.protocol.Datagram; +import qunar.tc.qmq.protocol.consumer.MetaInfoRequest; +import qunar.tc.qmq.protocol.consumer.MetaInfoRequestPayloadHolder; +import qunar.tc.qmq.util.RemotingBuilder; + +/** + * @author yiqun.fan create on 17-8-31. + */ +class MetaInfoClientNettyImpl extends AbstractNettyClient implements MetaInfoClient { + private static final Logger LOGGER = LoggerFactory.getLogger(MetaInfoClient.class); + + private MetaServerLocator locator; + + public static MetaInfoClientNettyImpl getClient() { + MetaInfoClientNettyImpl client = SUPPLIER.get(); + if (!client.isStarted()) { + NettyClientConfig config = new NettyClientConfig(); + config.setClientWorkerThreads(1); + client.start(config); + } + return client; + } + + private static final Supplier SUPPLIER = Suppliers.memoize(new Supplier() { + @Override + public MetaInfoClientNettyImpl get() { + return new MetaInfoClientNettyImpl(); + } + }); + + private MetaInfoClientNettyImpl() { + super("qmq-metaclient"); + } + + private MetaInfoClientHandler clientHandler; + + @Override + protected void initHandler() { + clientHandler = new MetaInfoClientHandler(); + } + + @Override + protected ChannelInitializer newChannelInitializer(final NettyClientConfig config, final DefaultEventExecutorGroup eventExecutors, final NettyConnectManageHandler connectManager) { + return new ChannelInitializer() { + @Override + protected void initChannel(SocketChannel ch) throws Exception { + ch.pipeline().addLast(eventExecutors, + new EncodeHandler(), + new DecodeHandler(false), + connectManager, + clientHandler); + } + }; + } + + @Override + public void sendRequest(final MetaInfoRequest request) { + try { + String metaServer = queryMetaServerAddress(); + if (metaServer == null) return; + final Channel channel = getOrCreateChannel(metaServer); + final Datagram datagram = RemotingBuilder.buildRequestDatagram(CommandCode.CLIENT_REGISTER, new MetaInfoRequestPayloadHolder(request)); + channel.writeAndFlush(datagram).addListener(new ChannelFutureListener() { + @Override + public void operationComplete(ChannelFuture future) throws Exception { + if (future.isSuccess()) { + LOGGER.debug("MetaInfoClientNettyImpl", "request meta info send success. {}", request); + } else { + LOGGER.debug("MetaInfoClientNettyImpl", "request meta info send fail. {}", request); + } + } + }); + } catch (Exception e) { + LOGGER.debug("MetaInfoClientNettyImpl", "request meta info exception. {}", request, e); + } + } + + private volatile String metaServer; + + private volatile long lastUpdate; + + private static final long UPDATE_INTERVAL = 1000 * 60; + + private String queryMetaServerAddress() { + if (metaServer == null) { + metaServer = queryMetaServerAddressWithRetry(); + lastUpdate = System.currentTimeMillis(); + return metaServer; + } + if (System.currentTimeMillis() - lastUpdate > UPDATE_INTERVAL) { + Optional optional = locator.queryEndpoint(); + if (optional.isPresent()) { + this.metaServer = optional.get(); + lastUpdate = System.currentTimeMillis(); + } + } + return metaServer; + } + + private String queryMetaServerAddressWithRetry() { + for (int i = 0; i < 3; ++i) { + Optional optional = locator.queryEndpoint(); + if (optional.isPresent()) + return optional.get(); + } + return null; + } + + @Override + public void registerResponseSubscriber(ResponseSubscriber subscriber) { + clientHandler.registerResponseSubscriber(subscriber); + } + + public void setMetaServerLocator(MetaServerLocator locator) { + this.locator = locator; + } +} diff --git a/qmq-client/src/main/java/qunar/tc/qmq/metainfoclient/MetaInfoService.java b/qmq-client/src/main/java/qunar/tc/qmq/metainfoclient/MetaInfoService.java new file mode 100644 index 00000000..01cd6cd0 --- /dev/null +++ b/qmq-client/src/main/java/qunar/tc/qmq/metainfoclient/MetaInfoService.java @@ -0,0 +1,298 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.metainfoclient; + +import com.google.common.base.Preconditions; +import com.google.common.base.Strings; +import com.google.common.eventbus.EventBus; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import qunar.tc.qmq.base.ClientRequestType; +import qunar.tc.qmq.base.OnOfflineState; +import qunar.tc.qmq.broker.BrokerClusterInfo; +import qunar.tc.qmq.broker.BrokerGroupInfo; +import qunar.tc.qmq.common.ClientType; +import qunar.tc.qmq.concurrent.NamedThreadFactory; +import qunar.tc.qmq.meta.BrokerCluster; +import qunar.tc.qmq.meta.BrokerGroup; +import qunar.tc.qmq.meta.BrokerState; +import qunar.tc.qmq.meta.MetaServerLocator; +import qunar.tc.qmq.metrics.Metrics; +import qunar.tc.qmq.protocol.consumer.MetaInfoRequest; +import qunar.tc.qmq.protocol.consumer.MetaInfoResponse; +import qunar.tc.qmq.utils.RetrySubjectUtils; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.locks.ReentrantLock; + +import static qunar.tc.qmq.metrics.MetricsConstants.SUBJECT_GROUP_ARRAY; + +/** + * @author yiqun.fan create on 17-8-31. + */ +public class MetaInfoService implements MetaInfoClient.ResponseSubscriber, Runnable { + private static final Logger LOGGER = LoggerFactory.getLogger(MetaInfoService.class); + + private static final long REFRESH_INTERVAL_SECONDS = 60; + + private final EventBus eventBus = new EventBus("meta-info"); + + private final ConcurrentHashMap metaInfoRequests = new ConcurrentHashMap<>(); + + private final ReentrantLock updateLock = new ReentrantLock(); + + private long lastUpdateTimestamp = -1; + + private final MetaInfoClient client; + + private ConsumerStateChangedListener consumerStateChangedListener; + + private String clientId; + private String metaServer; + + public MetaInfoService() { + this.client = MetaInfoClientNettyImpl.getClient(); + } + + public void init() { + Preconditions.checkNotNull(metaServer, "meta server必须提供"); + this.client.setMetaServerLocator(new MetaServerLocator(metaServer)); + this.client.registerResponseSubscriber(this); + Executors.newSingleThreadScheduledExecutor(new NamedThreadFactory("qmq-meta-refresh")) + .scheduleAtFixedRate(this, REFRESH_INTERVAL_SECONDS, REFRESH_INTERVAL_SECONDS, TimeUnit.SECONDS); + } + + public void register(Object subscriber) { + eventBus.register(subscriber); + } + + public boolean tryAddRequest(MetaInfoRequestParam param) { + return metaInfoRequests.put(param, 1) == null; + } + + @Override + public void run() { + for (Map.Entry entry : metaInfoRequests.entrySet()) { + requestWrapper(entry.getKey()); + } + } + + public void requestWrapper(MetaInfoRequestParam param) { + try { + Metrics.counter("qmq_pull_metainfo_request_count", SUBJECT_GROUP_ARRAY, new String[]{param.subject, param.group}).inc(); + request(param); + } catch (Exception e) { + LOGGER.debug("MetaInfoService", "request meta info exception. {} {} {}", param.clientType.name(), param.subject, param.group, e); + Metrics.counter("qmq_pull_metainfo_request_fail", SUBJECT_GROUP_ARRAY, new String[]{param.subject, param.group}).inc(); + } + } + + private void request(MetaInfoRequestParam param) { + MetaInfoRequest request = new MetaInfoRequest(); + request.setSubject(param.subject); + request.setClientType(param.clientType); + request.setClientId(this.clientId); + request.setConsumerGroup(param.group); + request.setAppCode(param.getAppCode()); + + if (tryAddRequest(param)) { + request.setRequestType(ClientRequestType.ONLINE); + } else { + request.setRequestType(ClientRequestType.HEARTBEAT); + } + + LOGGER.debug("MetaInfoServiceRequest", "meta info request: {}", request); + client.sendRequest(request); + } + + @Override + public void onResponse(MetaInfoResponse response) { + updateConsumerState(response); + + MetaInfo metaInfo = parseResponse(response); + if (metaInfo != null) { + if (metaInfo.getClusterInfo().getGroups().isEmpty() && metaInfo.getClientType() != ClientType.CONSUMER) { + LOGGER.debug("MetaInfoService", "meta server return empty broker, will retry in a few seconds. subject={}, client={}", metaInfo.getSubject(), metaInfo.getClientType().name()); + } else { + LOGGER.debug("MetaInfoService", "meta info: {}", metaInfo); + eventBus.post(metaInfo); + } + } else { + LOGGER.warn("request meta info fail, will retry in a few seconds."); + } + } + + private void updateConsumerState(MetaInfoResponse response) { + updateLock.lock(); + try { + if (isStale(response.getTimestamp(), lastUpdateTimestamp)) { + LOGGER.debug("MetaInfoService", "skip response {}", response); + return; + } + lastUpdateTimestamp = response.getTimestamp(); + + final String subject = response.getSubject(); + final String consumerGroup = response.getConsumerGroup(); + + if (RetrySubjectUtils.isRealSubject(subject) && !Strings.isNullOrEmpty(consumerGroup)) { + boolean online = response.getOnOfflineState() == OnOfflineState.ONLINE; + LOGGER.debug("MetaInfoService", "消费者状态发生变更 {}/{}:{}", subject, consumerGroup, online); + triggerConsumerStateChanged(subject, consumerGroup, online); + } + + } catch (Exception e) { + LOGGER.error("update meta info exception. response={}", response, e); + } finally { + updateLock.unlock(); + } + } + + private boolean isStale(long thisTimestamp, long lastUpdateTimestamp) { + return thisTimestamp < lastUpdateTimestamp; + } + + private MetaInfo parseResponse(MetaInfoResponse response) { + if (response == null) return null; + + String subject = response.getSubject(); + if (Strings.isNullOrEmpty(subject)) return null; + + ClientType clientType = parseClientType(response); + if (clientType == null) return null; + + BrokerCluster cluster = response.getBrokerCluster(); + List groups = cluster == null ? null : cluster.getBrokerGroups(); + if (groups == null || groups.isEmpty()) { + return new MetaInfo(subject, clientType, new BrokerClusterInfo()); + } + + List validBrokers = new ArrayList<>(groups.size()); + for (BrokerGroup group : groups) { + if (inValid(group)) continue; + + BrokerState state = group.getBrokerState(); + if (clientType.isConsumer() && state.canRead()) { + validBrokers.add(group); + } else if (clientType.isProducer() && state.canWrite()) { + validBrokers.add(group); + } + } + if (validBrokers.isEmpty()) { + return new MetaInfo(subject, clientType, new BrokerClusterInfo()); + } + List groupInfos = new ArrayList<>(validBrokers.size()); + for (int i = 0; i < validBrokers.size(); i++) { + BrokerGroup bg = validBrokers.get(i); + groupInfos.add(new BrokerGroupInfo(i, bg.getGroupName(), bg.getMaster(), bg.getSlaves())); + } + BrokerClusterInfo clusterInfo = new BrokerClusterInfo(groupInfos); + return new MetaInfo(subject, clientType, clusterInfo); + } + + private boolean inValid(BrokerGroup group) { + return group == null || Strings.isNullOrEmpty(group.getGroupName()) || Strings.isNullOrEmpty(group.getMaster()); + } + + private ClientType parseClientType(MetaInfoResponse response) { + return ClientType.of(response.getClientTypeCode()); + } + + public static MetaInfoRequestParam buildRequestParam(ClientType clientType, String subject, String group, String appCode) { + return new MetaInfoRequestParam(clientType, subject, group, appCode); + } + + public void setConsumerStateChangedListener(ConsumerStateChangedListener listener) { + this.consumerStateChangedListener = listener; + } + + private void triggerConsumerStateChanged(String subject, String consumerGroup, boolean online) { + if (this.consumerStateChangedListener == null) return; + + if (online) { + this.consumerStateChangedListener.online(subject, consumerGroup); + } else { + this.consumerStateChangedListener.offline(subject, consumerGroup); + } + } + + public void setClientId(String clientId) { + this.clientId = clientId; + } + + public void setMetaServer(String metaServer) { + this.metaServer = metaServer; + } + + public static final class MetaInfoRequestParam { + private final ClientType clientType; + private final String subject; + private final String group; + private final String appCode; + + MetaInfoRequestParam(ClientType clientType, String subject, String group, String appCode) { + this.clientType = clientType; + this.subject = Strings.nullToEmpty(subject); + this.group = Strings.nullToEmpty(group); + this.appCode = appCode; + } + + public ClientType getClientType() { + return clientType; + } + + public String getSubject() { + return subject; + } + + public String getGroup() { + return group; + } + + public String getAppCode() { + return appCode; + } + + @Override + public int hashCode() { + return clientType.getCode() + 31 * subject.hashCode() + 31 * group.hashCode(); + } + + @Override + public boolean equals(Object obj) { + if (this == obj) return true; + if (obj == null) return false; + if (!(obj instanceof MetaInfoRequestParam)) return false; + MetaInfoRequestParam param = (MetaInfoRequestParam) obj; + return clientType.getCode() == param.getClientType().getCode() + && subject.equals(param.subject) && group.equals(param.group); + } + + @Override + public String toString() { + return "MetaInfoParam{" + + "clientType=" + clientType.name() + + ", subject='" + subject + '\'' + + ", group='" + group + '\'' + + '}'; + } + } +} diff --git a/qmq-client/src/main/java/qunar/tc/qmq/producer/ConfigCenter.java b/qmq-client/src/main/java/qunar/tc/qmq/producer/ConfigCenter.java new file mode 100644 index 00000000..08502fcc --- /dev/null +++ b/qmq-client/src/main/java/qunar/tc/qmq/producer/ConfigCenter.java @@ -0,0 +1,82 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.producer; + +/** + * @author zhenyu.nie created on 2017 2017/7/5 11:57 + */ +public class ConfigCenter { + private static final ConfigCenter INSTANCE = new ConfigCenter(); + + public static ConfigCenter getInstance() { + return INSTANCE; + } + + private static final int MIN_EXPIRED_TIME = 15; + + private int maxQueueSize = 10000; + private int sendThreads = 3; + private int sendBatch = 30; + private long sendTimeoutMillis = 5000; + private int sendTryCount = 10; + + private boolean syncSend = false; + + public int getMaxQueueSize() { + return maxQueueSize; + } + + public void setMaxQueueSize(int maxQueueSize) { + this.maxQueueSize = maxQueueSize; + } + + public int getSendThreads() { + return sendThreads; + } + + public void setSendThreads(int sendThreads) { + this.sendThreads = sendThreads; + } + + public int getSendBatch() { + return sendBatch; + } + + public void setSendBatch(int sendBatch) { + this.sendBatch = sendBatch; + } + + public long getSendTimeoutMillis() { + return sendTimeoutMillis; + } + + public int getSendTryCount() { + return sendTryCount; + } + + public void setSendTryCount(int sendTryCount) { + this.sendTryCount = sendTryCount; + } + + public boolean isSyncSend() { + return syncSend; + } + + public int getMinExpiredTime() { + return MIN_EXPIRED_TIME; + } +} diff --git a/qmq-client/src/main/java/qunar/tc/qmq/producer/MessageProducerProvider.java b/qmq-client/src/main/java/qunar/tc/qmq/producer/MessageProducerProvider.java new file mode 100644 index 00000000..b6ec3944 --- /dev/null +++ b/qmq-client/src/main/java/qunar/tc/qmq/producer/MessageProducerProvider.java @@ -0,0 +1,183 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ +package qunar.tc.qmq.producer; + +import com.google.common.base.Preconditions; +import io.opentracing.Scope; +import io.opentracing.Tracer; +import io.opentracing.util.GlobalTracer; +import qunar.tc.qmq.Message; +import qunar.tc.qmq.MessageProducer; +import qunar.tc.qmq.MessageSendStateListener; +import qunar.tc.qmq.base.BaseMessage; +import qunar.tc.qmq.common.ClientIdProvider; +import qunar.tc.qmq.common.ClientIdProviderFactory; +import qunar.tc.qmq.producer.idgenerator.IdGenerator; +import qunar.tc.qmq.producer.idgenerator.TimestampAndHostIdGenerator; +import qunar.tc.qmq.producer.sender.NettyRouterManager; +import qunar.tc.qmq.tracing.TraceUtil; + +import javax.annotation.PostConstruct; +import javax.annotation.PreDestroy; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; + +/** + * @author miao.yang susing@gmail.com + * @date 2013-1-5 + */ +public class MessageProducerProvider implements MessageProducer { + private static final ConfigCenter configs = ConfigCenter.getInstance(); + + private final IdGenerator idGenerator; + + private final AtomicBoolean STARTED = new AtomicBoolean(false); + + private final NettyRouterManager routerManager; + + private ClientIdProvider clientIdProvider; + + private final Tracer tracer; + + private String appCode; + private String metaServer; + + /** + * 自动路由机房 + */ + public MessageProducerProvider() { + this.idGenerator = new TimestampAndHostIdGenerator(); + this.clientIdProvider = ClientIdProviderFactory.createDefault(); + this.routerManager = new NettyRouterManager(); + this.tracer = GlobalTracer.get(); + } + + @PostConstruct + public void init() { + Preconditions.checkNotNull(appCode, "appCode唯一标识一个应用"); + Preconditions.checkNotNull(metaServer, "metaServer的http地址"); + + this.routerManager.setMetaServer(this.metaServer); + + if (STARTED.compareAndSet(false, true)) { + routerManager.init(clientIdProvider.get()); + } + } + + @Override + public Message generateMessage(String subject) { + BaseMessage msg = new BaseMessage(idGenerator.getNext(), subject); + msg.setExpiredDelay(configs.getMinExpiredTime(), TimeUnit.MINUTES); + msg.setProperty(BaseMessage.keys.qmq_appCode, appCode); + return msg; + } + + @Override + public void sendMessage(Message message) { + sendMessage(message, null); + } + + @Override + public void sendMessage(Message message, MessageSendStateListener listener) { + if (!STARTED.get()) { + throw new RuntimeException("MessageProducerProvider未初始化,如果使用非Spring的方式请确认init()是否调用"); + } + try (Scope ignored = tracer.buildSpan("Qmq.Produce.Send") + .withTag("appCode", appCode) + .withTag("subject", message.getSubject()) + .withTag("messageId", message.getMessageId()) + .startActive(true)) { + ProduceMessageImpl pm = initProduceMessage(message, listener); + pm.send(); + } + } + + private ProduceMessageImpl initProduceMessage(Message message, MessageSendStateListener listener) { + BaseMessage base = (BaseMessage) message; + routerManager.validateMessage(message); + ProduceMessageImpl pm = new ProduceMessageImpl(base, routerManager.getSender()); + pm.setSendTryCount(configs.getSendTryCount()); + pm.setSendStateListener(listener); + pm.setSyncSend(configs.isSyncSend()); + + String value = routerManager.registryOf(message); + TraceUtil.setTag("registry", value, tracer); + return pm; + } + + @PreDestroy + public void destroy() { + routerManager.destroy(); + } + + /** + * 内存发送队列最大值,默认值 @see QUEUE_MEM_SIZE + * + * @param maxQueueSize 内存队列大小 + */ + public void setMaxQueueSize(int maxQueueSize) { + configs.setMaxQueueSize(maxQueueSize); + } + + /** + * 发送线程数 @see SEND_THREADS + * + * @param sendThreads + */ + public void setSendThreads(int sendThreads) { + configs.setSendThreads(sendThreads); + } + + /** + * 批量发送,每批量大小 @see SEND_BATCH + * + * @param sendBatch + */ + public void setSendBatch(int sendBatch) { + configs.setSendBatch(sendBatch); + } + + /** + * 发送失败重试次数 + * + * @param sendTryCount @see SEND_TRY_COUNT + */ + public void setSendTryCount(int sendTryCount) { + configs.setSendTryCount(sendTryCount); + } + + public void setClientIdProvider(ClientIdProvider clientIdProvider) { + this.clientIdProvider = clientIdProvider; + } + + /** + * 为了方便维护应用与消息主题之间的关系,每个应用提供一个唯一的标识 + * + * @param appCode + */ + public void setAppCode(String appCode) { + this.appCode = appCode; + } + + /** + * 用于发现meta server集群的地址 + * + * @param metaServer + */ + public void setMetaServer(String metaServer) { + this.metaServer = metaServer; + } +} \ No newline at end of file diff --git a/qmq-client/src/main/java/qunar/tc/qmq/producer/ProduceMessageImpl.java b/qmq-client/src/main/java/qunar/tc/qmq/producer/ProduceMessageImpl.java new file mode 100644 index 00000000..260d0fb6 --- /dev/null +++ b/qmq-client/src/main/java/qunar/tc/qmq/producer/ProduceMessageImpl.java @@ -0,0 +1,270 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ +package qunar.tc.qmq.producer; + +import io.opentracing.Scope; +import io.opentracing.Span; +import io.opentracing.Tracer; +import io.opentracing.util.GlobalTracer; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import qunar.tc.qmq.MessageSendStateListener; +import qunar.tc.qmq.ProduceMessage; +import qunar.tc.qmq.base.BaseMessage; +import qunar.tc.qmq.concurrent.NamedThreadFactory; +import qunar.tc.qmq.metrics.Metrics; +import qunar.tc.qmq.metrics.QmqCounter; +import qunar.tc.qmq.tracing.TraceUtil; + +import java.util.concurrent.*; +import java.util.concurrent.atomic.AtomicInteger; + +/** + * @author miao.yang susing@gmail.com + * @date 2013-1-5 + */ +class ProduceMessageImpl implements ProduceMessage { + private static final Logger LOGGER = LoggerFactory.getLogger(ProduceMessageImpl.class); + + private static final int INIT = 0; + private static final int QUEUED = 1; + + private static final int FINISH = 100; + private static final int ERROR = -1; + private static final int BLOCK = -2; + + private static final int DEFAULT_THREADS = 10; + + private static final int DEFAULT_QUEUE_SIZE = 1000; + + private static final Executor EXECUTOR; + + private static final QmqCounter sendCount = Metrics.counter("qmq_client_send_count"); + private static final QmqCounter sendOkCount = Metrics.counter("qmq_client_send_ok_count"); + private static final QmqCounter sendErrorCount = Metrics.counter("qmq_client_send_error_count"); + private static final QmqCounter sendFailCount = Metrics.counter("qmq_client_send_fail_count"); + private static final QmqCounter resendCount = Metrics.counter("qmq_client_resend_count"); + private static final QmqCounter enterQueueFail = Metrics.counter("qmq_client_enter_queue_fail"); + + static { + EXECUTOR = new ThreadPoolExecutor(1, DEFAULT_THREADS, 1, TimeUnit.MINUTES, + new LinkedBlockingQueue(DEFAULT_QUEUE_SIZE), + new NamedThreadFactory("default-send-listener", true), + new RejectedExecutionHandler() { + @Override + public void rejectedExecution(Runnable r, ThreadPoolExecutor executor) { + LOGGER.error("MessageSendStateListener任务被拒绝,现在的大小为:threads-{}, queue size-{}.如果在该listener里执行了比较重的操作", DEFAULT_THREADS, DEFAULT_QUEUE_SIZE); + } + }); + + } + + /** + * 最多尝试次数 + */ + private int sendTryCount; + + private final BaseMessage base; + + private final QueueSender sender; + + //tracer + private final Tracer tracer; + private Span traceSpan; + private Scope traceScope; + + private MessageSendStateListener sendStateListener; + + private final AtomicInteger state = new AtomicInteger(INIT); + private final AtomicInteger tries = new AtomicInteger(0); + private boolean syncSend; + + public ProduceMessageImpl(BaseMessage base, QueueSender sender) { + this.base = base; + this.sender = sender; + this.tracer = GlobalTracer.get(); + } + + @Override + public void send() { + sendCount.inc(); + attachTraceData(); + doSend(); + } + + private void doSend() { + if (state.compareAndSet(INIT, QUEUED)) { + tries.incrementAndGet(); + if (sendSync()) return; + + try (Scope scope = tracer.buildSpan("Qmq.QueueSender.Send").startActive(false)) { + traceSpan = scope.span(); + + if (sender.offer(this)) { + LOGGER.info("进入发送队列 {}:{}", getSubject(), getMessageId()); + } else { + enterQueueFail.inc(); + LOGGER.info("内存发送队列已满! 此消息在用户进程阻塞,等待队列激活 {}:{}", getSubject(), getMessageId()); + if (sender.offer(this, 50)) { + LOGGER.info("进入发送队列 {}:{}", getSubject(), getMessageId()); + } else { + LOGGER.info("由于无法入队,发送失败!取消发送 {}:{}", getSubject(), getMessageId()); + onFailed(); + } + } + } + + } else { + enterQueueFail.inc(); + throw new IllegalStateException("同一条消息不能被入队两次."); + } + } + + private boolean sendSync() { + if (!syncSend) return false; + sender.send(this); + return true; + } + + @Override + public void finish() { + state.set(FINISH); + onSuccess(); + closeTrace(); + } + + private void onSuccess() { + sendOkCount.inc(); + if (sendStateListener == null) return; + EXECUTOR.execute(new Runnable() { + @Override + public void run() { + sendStateListener.onSuccess(base); + } + }); + } + + @Override + public void error(Exception e) { + state.set(ERROR); + try { + if (tries.get() < sendTryCount) { + sendErrorCount.inc(); + LOGGER.info("发送失败, 重新发送. tryCount: {} {}:{}", tries.get(), getSubject(), getMessageId()); + resend(); + } else { + failed(); + } + } finally { + closeTrace(); + } + } + + @Override + public void block() { + try { + state.set(BLOCK); + LOGGER.info("消息被拒绝 {}:{}", getSubject(), getMessageId()); + if (syncSend) { + throw new RuntimeException("消息被拒绝且没有store可恢复,请检查应用授权配置"); + } + } finally { + onFailed(); + closeTrace(); + } + } + + @Override + public void failed() { + state.set(ERROR); + try { + sendErrorCount.inc(); + String message = "发送失败, 已尝试" + tries.get() + "次不再尝试重新发送."; + LOGGER.info(message); + + if (syncSend) { + throw new RuntimeException(message); + } + } finally { + onFailed(); + closeTrace(); + } + } + + private void onFailed() { + TraceUtil.recordEvent("send_failed", tracer); + sendFailCount.inc(); + if (sendStateListener == null) return; + EXECUTOR.execute(new Runnable() { + @Override + public void run() { + sendStateListener.onFailed(base); + } + }); + } + + private void resend() { + resendCount.inc(); + traceSpan = null; + traceScope = null; + state.set(INIT); + TraceUtil.recordEvent("retry", tracer); + doSend(); + } + + @Override + public String getMessageId() { + return base.getMessageId(); + } + + @Override + public String getSubject() { + return base.getSubject(); + } + + @Override + public void startSendTrace() { + if (traceSpan == null) return; + traceScope = tracer.scopeManager().activate(traceSpan, false); + attachTraceData(); + } + + private void attachTraceData() { + TraceUtil.inject(base, tracer); + } + + private void closeTrace() { + if (traceScope == null) return; + traceScope.close(); + } + + @Override + public BaseMessage getBase() { + return base; + } + + public void setSendTryCount(int sendTryCount) { + this.sendTryCount = sendTryCount; + } + + public void setSendStateListener(MessageSendStateListener sendStateListener) { + this.sendStateListener = sendStateListener; + } + + public void setSyncSend(boolean syncSend) { + this.syncSend = syncSend; + } +} diff --git a/qmq-client/src/main/java/qunar/tc/qmq/producer/QueueSender.java b/qmq-client/src/main/java/qunar/tc/qmq/producer/QueueSender.java new file mode 100644 index 00000000..8c8dfcb7 --- /dev/null +++ b/qmq-client/src/main/java/qunar/tc/qmq/producer/QueueSender.java @@ -0,0 +1,34 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ +package qunar.tc.qmq.producer; + + +import qunar.tc.qmq.ProduceMessage; + +/** + * @author miao.yang susing@gmail.com + * @date 2013-1-6 + */ +public interface QueueSender { + + boolean offer(ProduceMessage pm); + + boolean offer(ProduceMessage pm, long millisecondWait); + + void send(ProduceMessage pm); + + void destroy(); +} diff --git a/qmq-client/src/main/java/qunar/tc/qmq/producer/RegistryResolver.java b/qmq-client/src/main/java/qunar/tc/qmq/producer/RegistryResolver.java new file mode 100644 index 00000000..c218f7ff --- /dev/null +++ b/qmq-client/src/main/java/qunar/tc/qmq/producer/RegistryResolver.java @@ -0,0 +1,34 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.producer; + +import java.util.Set; + +/** + * User: zhaohuiyu + * Date: 10/28/14 + * Time: 2:30 PM + */ +public interface RegistryResolver { + String resolve(); + + String resolve(String dataCenter); + + String dataCenter(); + + Set list(); +} diff --git a/qmq-client/src/main/java/qunar/tc/qmq/producer/SendErrorHandler.java b/qmq-client/src/main/java/qunar/tc/qmq/producer/SendErrorHandler.java new file mode 100644 index 00000000..1a50ad4d --- /dev/null +++ b/qmq-client/src/main/java/qunar/tc/qmq/producer/SendErrorHandler.java @@ -0,0 +1,34 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.producer; + +import qunar.tc.qmq.ProduceMessage; +import qunar.tc.qmq.service.exceptions.MessageException; + +/** + * @author zhenyu.nie created on 2017 2017/7/5 17:23 + */ +public interface SendErrorHandler { + + void error(ProduceMessage pm, Exception e); + + void failed(ProduceMessage pm, Exception e); + + void block(ProduceMessage pm, MessageException ex); + + void finish(ProduceMessage pm, Exception e); +} diff --git a/qmq-client/src/main/java/qunar/tc/qmq/producer/idgenerator/IdGenerator.java b/qmq-client/src/main/java/qunar/tc/qmq/producer/idgenerator/IdGenerator.java new file mode 100644 index 00000000..37a1c351 --- /dev/null +++ b/qmq-client/src/main/java/qunar/tc/qmq/producer/idgenerator/IdGenerator.java @@ -0,0 +1,26 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.producer.idgenerator; + +/** + * User: zhaohuiyu + * Date: 6/4/13 + * Time: 10:06 AM + */ +public interface IdGenerator { + String getNext(); +} diff --git a/qmq-client/src/main/java/qunar/tc/qmq/producer/idgenerator/TimestampAndHostIdGenerator.java b/qmq-client/src/main/java/qunar/tc/qmq/producer/idgenerator/TimestampAndHostIdGenerator.java new file mode 100644 index 00000000..7f0be5b6 --- /dev/null +++ b/qmq-client/src/main/java/qunar/tc/qmq/producer/idgenerator/TimestampAndHostIdGenerator.java @@ -0,0 +1,52 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.producer.idgenerator; + +import qunar.tc.qmq.utils.NetworkUtils; +import qunar.tc.qmq.utils.PidUtil; + +import java.sql.Timestamp; +import java.util.concurrent.atomic.AtomicInteger; + +/** + * User: zhaohuiyu + * Date: 6/4/13 + * Time: 10:06 AM + */ +public class TimestampAndHostIdGenerator implements IdGenerator { + private static final int[] codex = {2, 3, 5, 6, 8, 9, 19, 11, 12, 14, 15, 17, 18}; + private static final AtomicInteger messageOrder = new AtomicInteger(0); + + private static final String localAddress = NetworkUtils.getLocalAddress(); + + //在生成message id的时候带上进程id,避免一台机器上部署多个服务都发同样的消息时出问题 + private static final int PID = PidUtil.getPid(); + + @Override + public String getNext() { + StringBuilder sb = new StringBuilder(40); + long time = System.currentTimeMillis(); + String ts = new Timestamp(time).toString(); + + for (int idx : codex) + sb.append(ts.charAt(idx)); + sb.append('.').append(localAddress); + sb.append('.').append(PID); + sb.append('.').append(messageOrder.getAndIncrement()); //可能为负数.但是无所谓. + return sb.toString(); + } +} diff --git a/qmq-client/src/main/java/qunar/tc/qmq/producer/sender/AbstractRouterManager.java b/qmq-client/src/main/java/qunar/tc/qmq/producer/sender/AbstractRouterManager.java new file mode 100644 index 00000000..1753d9c9 --- /dev/null +++ b/qmq-client/src/main/java/qunar/tc/qmq/producer/sender/AbstractRouterManager.java @@ -0,0 +1,79 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.producer.sender; + +import com.google.common.base.Preconditions; +import qunar.tc.qmq.Message; +import qunar.tc.qmq.producer.ConfigCenter; +import qunar.tc.qmq.producer.QueueSender; + +import java.util.concurrent.atomic.AtomicBoolean; + +/** + * Created by zhaohui.yu + * 9/13/17 + */ +abstract class AbstractRouterManager implements RouterManager { + + private static final ConfigCenter configs = ConfigCenter.getInstance(); + + private Router router; + + private QueueSender sender; + + private final AtomicBoolean STARTED = new AtomicBoolean(false); + + @Override + public void init(String clientId) { + if (STARTED.compareAndSet(false, true)) { + doInit(clientId); + this.sender = new RPCQueueSender("qmq-sender", configs.getMaxQueueSize(), configs.getSendThreads(), configs.getSendBatch(), this); + } + } + + protected void doInit(String clientId) { + + } + + @Override + public String registryOf(Message message) { + return router.route(message).url(); + } + + void setRouter(Router router) { + this.router = router; + } + + @Override + public QueueSender getSender() { + return sender; + } + + @Override + public Connection routeOf(Message message) { + Connection connection = router.route(message); + Preconditions.checkState(connection != NopRoute.NOP_CONNECTION, "与broker连接失败,可能是配置错误,请联系TCDev"); + return connection; + } + + @Override + public void destroy() { + if (sender != null) { + sender.destroy(); + } + } +} diff --git a/qmq-client/src/main/java/qunar/tc/qmq/producer/sender/Connection.java b/qmq-client/src/main/java/qunar/tc/qmq/producer/sender/Connection.java new file mode 100644 index 00000000..156c4a26 --- /dev/null +++ b/qmq-client/src/main/java/qunar/tc/qmq/producer/sender/Connection.java @@ -0,0 +1,38 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.producer.sender; + +import qunar.tc.qmq.ProduceMessage; +import qunar.tc.qmq.netty.exception.BrokerRejectException; +import qunar.tc.qmq.netty.exception.ClientSendException; +import qunar.tc.qmq.netty.exception.RemoteException; +import qunar.tc.qmq.service.exceptions.MessageException; + +import java.util.List; +import java.util.Map; + +/** + * @author zhenyu.nie created on 2017 2017/7/3 12:25 + */ +public interface Connection { + + String url(); + + Map send(List messages) throws RemoteException, ClientSendException, BrokerRejectException; + + void destroy(); +} diff --git a/qmq-client/src/main/java/qunar/tc/qmq/producer/sender/MessageSenderGroup.java b/qmq-client/src/main/java/qunar/tc/qmq/producer/sender/MessageSenderGroup.java new file mode 100644 index 00000000..0c1c3f43 --- /dev/null +++ b/qmq-client/src/main/java/qunar/tc/qmq/producer/sender/MessageSenderGroup.java @@ -0,0 +1,88 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.producer.sender; + +import qunar.tc.qmq.ProduceMessage; +import qunar.tc.qmq.producer.SendErrorHandler; +import qunar.tc.qmq.service.exceptions.BlockMessageException; +import qunar.tc.qmq.service.exceptions.DuplicateMessageException; +import qunar.tc.qmq.service.exceptions.MessageException; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Map; + +/** + * @author zhenyu.nie created on 2017 2017/7/5 17:26 + */ +class MessageSenderGroup { + + private final Connection connection; + private final SendErrorHandler errorHandler; + + private final List source; + + MessageSenderGroup(SendErrorHandler errorHandler, Connection connection) { + this.errorHandler = errorHandler; + this.connection = connection; + this.source = new ArrayList<>(); + } + + public void send() { + Map map; + try { + map = connection.send(source); + } catch (Exception e) { + for (ProduceMessage pm : source) { + errorHandler.error(pm, e); + } + return; + } + + if (map == null) { + for (ProduceMessage pm : source) { + errorHandler.error(pm, new MessageException(pm.getMessageId(), "return null")); + } + return; + } + + if (map.isEmpty()) + map = Collections.emptyMap(); + + for (ProduceMessage pm : source) { + MessageException ex = map.get(pm.getMessageId()); + if (ex == null || ex instanceof DuplicateMessageException) { + errorHandler.finish(pm, ex); + } else { + //如果是消息被拒绝,说明broker已经限速,不立即重试; + if (ex.isBrokerBusy()) { + errorHandler.failed(pm, ex); + } else if (ex instanceof BlockMessageException) { + //如果是block的,证明还没有被授权,也不重试,task也不重试,需要手工恢复 + errorHandler.block(pm, ex); + } else { + errorHandler.error(pm, ex); + } + } + } + } + + void addMessage(ProduceMessage source) { + this.source.add(source); + } +} diff --git a/qmq-client/src/main/java/qunar/tc/qmq/producer/sender/NettyConnection.java b/qmq-client/src/main/java/qunar/tc/qmq/producer/sender/NettyConnection.java new file mode 100644 index 00000000..a2a0ac53 --- /dev/null +++ b/qmq-client/src/main/java/qunar/tc/qmq/producer/sender/NettyConnection.java @@ -0,0 +1,196 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.producer.sender; + +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; +import io.netty.buffer.ByteBuf; +import qunar.tc.qmq.ProduceMessage; +import qunar.tc.qmq.base.BaseMessage; +import qunar.tc.qmq.broker.BrokerClusterInfo; +import qunar.tc.qmq.broker.BrokerGroupInfo; +import qunar.tc.qmq.broker.BrokerLoadBalance; +import qunar.tc.qmq.broker.BrokerService; +import qunar.tc.qmq.broker.impl.PollBrokerLoadBalance; +import qunar.tc.qmq.common.ClientType; +import qunar.tc.qmq.metrics.Metrics; +import qunar.tc.qmq.metrics.QmqCounter; +import qunar.tc.qmq.metrics.QmqTimer; +import qunar.tc.qmq.netty.exception.*; +import qunar.tc.qmq.protocol.*; +import qunar.tc.qmq.protocol.producer.MessageProducerCode; +import qunar.tc.qmq.protocol.producer.SendResult; +import qunar.tc.qmq.service.exceptions.BlockMessageException; +import qunar.tc.qmq.service.exceptions.DuplicateMessageException; +import qunar.tc.qmq.service.exceptions.MessageException; +import qunar.tc.qmq.tracing.TraceUtil; +import qunar.tc.qmq.util.RemotingBuilder; + +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.concurrent.TimeUnit; + +import static qunar.tc.qmq.metrics.MetricsConstants.SUBJECT_ARRAY; + +/** + * @author zhenyu.nie created on 2017 2017/7/5 15:08 + */ +class NettyConnection implements Connection { + private final String subject; + private final ClientType clientType; + private final NettyProducerClient producerClient; + private final BrokerService brokerService; + + private volatile BrokerGroupInfo lastSentBroker; + + private final BrokerLoadBalance brokerLoadBalance = PollBrokerLoadBalance.getInstance(); + + private final QmqCounter sendMessageCountMetrics; + private final QmqTimer sendMessageTimerMetrics; + + NettyConnection(String subject, ClientType clientType, NettyProducerClient producerClient, BrokerService brokerService) { + this.subject = subject; + this.clientType = clientType; + this.producerClient = producerClient; + this.brokerService = brokerService; + + sendMessageCountMetrics = Metrics.counter("qmq_client_send_msg_count", SUBJECT_ARRAY, new String[]{subject}); + sendMessageTimerMetrics = Metrics.timer("qmq_client_send_msg_timer"); + } + + public void init() { + brokerService.refresh(clientType, subject); + } + + @Override + public Map send(List messages) throws ClientSendException, RemoteException, BrokerRejectException { + sendMessageCountMetrics.inc(messages.size()); + long start = System.currentTimeMillis(); + try { + BrokerClusterInfo cluster = brokerService.getClusterBySubject(clientType, subject); + BrokerGroupInfo target = brokerLoadBalance.loadBalance(cluster, lastSentBroker); + if (target == null) { + throw new ClientSendException(ClientSendException.SendErrorCode.CREATE_CHANNEL_FAIL); + } + + lastSentBroker = target; + Datagram response = doSend(target, messages); + RemotingHeader responseHeader = response.getHeader(); + int code = responseHeader.getCode(); + switch (code) { + case CommandCode.SUCCESS: + return process(target, response); + case CommandCode.BROKER_REJECT: + handleSendReject(target); + throw new BrokerRejectException(""); + default: + throw new RemoteException(); + } + } finally { + sendMessageTimerMetrics.update(System.currentTimeMillis() - start, TimeUnit.MILLISECONDS); + } + } + + private void handleSendReject(BrokerGroupInfo target) { + if (target != null) { + target.setAvailable(false); + } + this.brokerService.refresh(ClientType.PRODUCER, subject); + } + + private Map process(BrokerGroupInfo target, Datagram response) throws RemoteResponseUnreadableException { + ByteBuf buf = response.getBody(); + try { + if (buf == null || !buf.isReadable()) { + return Collections.emptyMap(); + } + Map resultMap = QMQSerializer.deserializeSendResultMap(buf); + boolean brokerReject = false; + Map map = Maps.newHashMapWithExpectedSize(resultMap.size()); + for (Map.Entry entry : resultMap.entrySet()) { + String messageId = entry.getKey(); + SendResult result = entry.getValue(); + switch (result.getCode()) { + case MessageProducerCode.SUCCESS: + break; + case MessageProducerCode.MESSAGE_DUPLICATE: + map.put(messageId, new DuplicateMessageException(messageId)); + break; + case MessageProducerCode.BROKER_BUSY: + map.put(messageId, new MessageException(messageId, MessageException.BROKER_BUSY)); + break; + case MessageProducerCode.BROKER_READ_ONLY: + brokerReject = true; + map.put(messageId, new BrokerRejectException(messageId)); + break; + case MessageProducerCode.SUBJECT_NOT_ASSIGNED: + map.put(messageId, new SubjectNotAssignedException(messageId)); + break; + case MessageProducerCode.BLOCK: + map.put(messageId, new BlockMessageException(messageId)); + break; + default: + map.put(messageId, new MessageException(messageId, result.getRemark())); + break; + } + } + + if (brokerReject) { + handleSendReject(target); + } + return map; + } finally { + response.release(); + } + } + + private Datagram doSend(BrokerGroupInfo target, List messages) throws ClientSendException, RemoteTimeoutException { + try { + Datagram datagram = buildDatagram(messages); + TraceUtil.setTag("broker", target.getGroupName()); + Datagram result = producerClient.sendMessage(target, datagram); + target.markSuccess(); + return result; + } catch (ClientSendException | RemoteTimeoutException e) { + target.markFailed(); + Metrics.counter("qmq_client_send_msg_error").inc(messages.size()); + throw e; + } catch (Exception e) { + target.markFailed(); + Metrics.counter("qmq_client_send_msg_error").inc(messages.size()); + throw new RuntimeException(e); + } + } + + private Datagram buildDatagram(List messages) { + final List baseMessages = Lists.newArrayListWithCapacity(messages.size()); + for (ProduceMessage message : messages) { + baseMessages.add((BaseMessage) message.getBase()); + } + return RemotingBuilder.buildRequestDatagram(CommandCode.SEND_MESSAGE, new MessagesPayloadHolder(baseMessages)); + } + + @Override + public String url() { + return "newqmq://" + subject; + } + + @Override + public void destroy() { + } +} diff --git a/qmq-client/src/main/java/qunar/tc/qmq/producer/sender/NettyProducerClient.java b/qmq-client/src/main/java/qunar/tc/qmq/producer/sender/NettyProducerClient.java new file mode 100644 index 00000000..2f95bee8 --- /dev/null +++ b/qmq-client/src/main/java/qunar/tc/qmq/producer/sender/NettyProducerClient.java @@ -0,0 +1,56 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.producer.sender; + +import qunar.tc.qmq.broker.BrokerGroupInfo; +import qunar.tc.qmq.config.NettyClientConfigManager; +import qunar.tc.qmq.netty.client.NettyClient; +import qunar.tc.qmq.netty.exception.ClientSendException; +import qunar.tc.qmq.netty.exception.RemoteTimeoutException; +import qunar.tc.qmq.producer.ConfigCenter; +import qunar.tc.qmq.protocol.Datagram; + +/** + * @author zhenyu.nie created on 2017 2017/7/4 17:49 + */ +class NettyProducerClient { + + private static final ConfigCenter CONFIG = ConfigCenter.getInstance(); + + private volatile boolean start = false; + + private NettyClient client; + + NettyProducerClient() { + client = NettyClient.getClient(); + } + + public synchronized void start() { + if (start) { + return; + } + + client.start(NettyClientConfigManager.get().getDefaultClientConfig()); + start = true; + } + + Datagram sendMessage(BrokerGroupInfo group, Datagram datagram) throws InterruptedException, ClientSendException, RemoteTimeoutException { + return client.sendSync(group.getMaster(), datagram, CONFIG.getSendTimeoutMillis()); + } + + +} diff --git a/qmq-client/src/main/java/qunar/tc/qmq/producer/sender/NettyRouter.java b/qmq-client/src/main/java/qunar/tc/qmq/producer/sender/NettyRouter.java new file mode 100644 index 00000000..01a7ce38 --- /dev/null +++ b/qmq-client/src/main/java/qunar/tc/qmq/producer/sender/NettyRouter.java @@ -0,0 +1,58 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.producer.sender; + +import qunar.tc.qmq.Message; +import qunar.tc.qmq.broker.BrokerService; +import qunar.tc.qmq.common.ClientType; +import qunar.tc.qmq.utils.DelayUtil; + +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; + +/** + * @author zhenyu.nie created on 2017 2017/7/5 16:29 + */ +class NettyRouter implements Router { + + private final NettyProducerClient producerClient; + private final BrokerService brokerService; + + private final ConcurrentMap cached; + + NettyRouter(NettyProducerClient producerClient, BrokerService brokerService) { + this.producerClient = producerClient; + this.brokerService = brokerService; + this.cached = new ConcurrentHashMap<>(); + } + + @Override + public Connection route(Message message) { + ClientType clientType = DelayUtil.isDelayMessage(message) ? ClientType.DELAY_PRODUCER : ClientType.PRODUCER; + String key = clientType.getCode() + "|" + message.getSubject(); + NettyConnection connection = cached.get(key); + if (connection != null) return connection; + + connection = new NettyConnection(message.getSubject(), clientType, producerClient, brokerService); + NettyConnection old = cached.putIfAbsent(key, connection); + if (old == null) { + connection.init(); + return connection; + } + return old; + } +} diff --git a/qmq-client/src/main/java/qunar/tc/qmq/producer/sender/NettyRouterManager.java b/qmq-client/src/main/java/qunar/tc/qmq/producer/sender/NettyRouterManager.java new file mode 100644 index 00000000..8776d738 --- /dev/null +++ b/qmq-client/src/main/java/qunar/tc/qmq/producer/sender/NettyRouterManager.java @@ -0,0 +1,90 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.producer.sender; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import qunar.tc.qmq.Message; +import qunar.tc.qmq.broker.BrokerService; +import qunar.tc.qmq.broker.impl.BrokerServiceImpl; +import qunar.tc.qmq.metainfoclient.MetaInfoService; +import qunar.tc.qmq.tracing.TraceUtil; + +import java.util.Map; + +/** + * @author zhenyu.nie created on 2017 2017/7/3 14:21 + */ +public class NettyRouterManager extends AbstractRouterManager { + private static final Logger logger = LoggerFactory.getLogger(NettyRouterManager.class); + + private static final int _32K = (32 * 1024) / 4; + + private final MetaInfoService metaInfoService; + private final BrokerService brokerService; + + private String metaServer; + private String appCode; + + public NettyRouterManager() { + this.metaInfoService = new MetaInfoService(); + this.brokerService = new BrokerServiceImpl(this.metaInfoService); + } + + @Override + public void doInit(String clientId) { + this.brokerService.setAppCode(appCode); + + this.metaInfoService.setMetaServer(metaServer); + this.metaInfoService.setClientId(clientId); + this.metaInfoService.init(); + + NettyProducerClient producerClient = new NettyProducerClient(); + producerClient.start(); + setRouter(new NettyRouter(producerClient, this.brokerService)); + } + + @Override + public String name() { + return "netty"; + } + + @Override + public void validateMessage(Message message) { + Map attrs = message.getAttrs(); + if (attrs == null) return; + for (Map.Entry entry : attrs.entrySet()) { + if (entry.getValue() == null) return; + if (!(entry.getValue() instanceof String)) return; + + String value = (String) entry.getValue(); + if (value.length() > _32K) { + TraceUtil.recordEvent("big_message"); + String msg = entry.getKey() + "的value长度超过32K,请使用Message.setLargeString方法设置,并且使用Message.getLargeString方法获取"; + logger.error(msg, new RuntimeException()); + } + } + } + + public void setMetaServer(String metaServer) { + this.metaServer = metaServer; + } + + public void setAppCode(String appCode) { + this.appCode = appCode; + } +} diff --git a/qmq-client/src/main/java/qunar/tc/qmq/producer/sender/NopRoute.java b/qmq-client/src/main/java/qunar/tc/qmq/producer/sender/NopRoute.java new file mode 100644 index 00000000..ae2388da --- /dev/null +++ b/qmq-client/src/main/java/qunar/tc/qmq/producer/sender/NopRoute.java @@ -0,0 +1,58 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.producer.sender; + +import com.google.common.collect.ImmutableMap; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import qunar.tc.qmq.ProduceMessage; +import qunar.tc.qmq.service.exceptions.MessageException; + +import java.util.List; +import java.util.Map; + +/** + * @author zhenyu.nie created on 2017 2017/7/3 12:48 + */ +class NopRoute implements Route { + + private static final Logger logger = LoggerFactory.getLogger(NopRoute.class); + + public static final Connection NOP_CONNECTION = new Connection() { + + @Override + public String url() { + return ""; + } + + @Override + public Map send(List messages) { + logger.warn("send message to nop route, {}", messages); + return ImmutableMap.of(); + } + + @Override + public void destroy() { + + } + }; + + @Override + public Connection route() { + return NOP_CONNECTION; + } +} diff --git a/qmq-client/src/main/java/qunar/tc/qmq/producer/sender/RPCQueueSender.java b/qmq-client/src/main/java/qunar/tc/qmq/producer/sender/RPCQueueSender.java new file mode 100644 index 00000000..701353eb --- /dev/null +++ b/qmq-client/src/main/java/qunar/tc/qmq/producer/sender/RPCQueueSender.java @@ -0,0 +1,146 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ +package qunar.tc.qmq.producer.sender; + +import com.google.common.collect.Maps; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import qunar.tc.qmq.ProduceMessage; +import qunar.tc.qmq.batch.BatchExecutor; +import qunar.tc.qmq.batch.Processor; +import qunar.tc.qmq.metrics.Metrics; +import qunar.tc.qmq.metrics.QmqTimer; +import qunar.tc.qmq.netty.exception.SubjectNotAssignedException; +import qunar.tc.qmq.producer.QueueSender; +import qunar.tc.qmq.producer.SendErrorHandler; +import qunar.tc.qmq.service.exceptions.MessageException; +import qunar.tc.qmq.tracing.TraceUtil; + +import java.util.Arrays; +import java.util.Collection; +import java.util.List; +import java.util.Map; +import java.util.concurrent.TimeUnit; + +/** + * @author miao.yang susing@gmail.com + * @date 2013-1-6 + */ +class RPCQueueSender implements QueueSender, SendErrorHandler, Processor { + + private static final Logger LOGGER = LoggerFactory.getLogger(RPCQueueSender.class); + + private final BatchExecutor executor; + + private final RouterManager routerManager; + + private final QmqTimer timer; + + public RPCQueueSender(String name, int maxQueueSize, int sendThreads, int sendBatch, RouterManager routerManager) { + this.routerManager = routerManager; + this.timer = Metrics.timer("qmq_client_send_task_timer"); + + this.executor = new BatchExecutor(name, sendBatch, this); + this.executor.setQueueSize(maxQueueSize); + this.executor.setThreads(sendThreads); + this.executor.init(); + } + + @Override + public boolean offer(ProduceMessage pm) { + return this.executor.addItem(pm); + } + + @Override + public boolean offer(ProduceMessage pm, long millisecondWait) { + boolean inserted; + try { + inserted = this.executor.addItem(pm, millisecondWait, TimeUnit.MILLISECONDS); + } catch (InterruptedException e) { + return false; + } + return inserted; + } + + @Override + public void send(ProduceMessage pm) { + process(Arrays.asList(pm)); + } + + @Override + public void process(List list) { + long start = System.currentTimeMillis(); + try { + //按照路由分组发送 + Collection messages = groupBy(list); + for (MessageSenderGroup group : messages) { + group.send(); + } + } finally { + timer.update(System.currentTimeMillis() - start, TimeUnit.MILLISECONDS); + } + } + + private Collection groupBy(List list) { + Map map = Maps.newHashMap(); + for (int i = 0; i < list.size(); ++i) { + ProduceMessage produceMessage = list.get(i); + produceMessage.startSendTrace(); + Connection connection = routerManager.routeOf(produceMessage.getBase()); + MessageSenderGroup group = map.get(connection); + if (group == null) { + group = new MessageSenderGroup(this, connection); + map.put(connection, group); + } + group.addMessage(produceMessage); + } + return map.values(); + } + + @Override + public void error(ProduceMessage pm, Exception e) { + if (!(e instanceof SubjectNotAssignedException)) { + LOGGER.warn("Message 发送失败! {}", pm.getMessageId(), e); + } + TraceUtil.recordEvent("error"); + pm.error(e); + } + + @Override + public void failed(ProduceMessage pm, Exception e) { + LOGGER.warn("Message 发送失败! {}", pm.getMessageId(), e); + TraceUtil.recordEvent("failed "); + pm.failed(); + } + + @Override + public void block(ProduceMessage pm, MessageException ex) { + LOGGER.warn("Message 发送失败! {},被server拒绝,请检查应用授权配置,如果需要恢复消息请手工到db恢复状态", pm.getMessageId(), ex); + TraceUtil.recordEvent("block"); + pm.block(); + } + + @Override + public void finish(ProduceMessage pm, Exception e) { + LOGGER.info("发送成功 {}:{}", pm.getSubject(), pm.getMessageId()); + pm.finish(); + } + + @Override + public void destroy() { + executor.destroy(); + } +} diff --git a/qmq-client/src/main/java/qunar/tc/qmq/producer/sender/Route.java b/qmq-client/src/main/java/qunar/tc/qmq/producer/sender/Route.java new file mode 100644 index 00000000..ca2659ea --- /dev/null +++ b/qmq-client/src/main/java/qunar/tc/qmq/producer/sender/Route.java @@ -0,0 +1,25 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.producer.sender; + +/** + * @author zhenyu.nie created on 2017 2017/7/3 12:24 + */ +public interface Route { + + Connection route(); +} diff --git a/qmq-client/src/main/java/qunar/tc/qmq/producer/sender/Router.java b/qmq-client/src/main/java/qunar/tc/qmq/producer/sender/Router.java new file mode 100644 index 00000000..6b983eed --- /dev/null +++ b/qmq-client/src/main/java/qunar/tc/qmq/producer/sender/Router.java @@ -0,0 +1,24 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.producer.sender; + +import qunar.tc.qmq.Message; + +public interface Router { + + Connection route(Message message); +} diff --git a/qmq-client/src/main/java/qunar/tc/qmq/producer/sender/RouterManager.java b/qmq-client/src/main/java/qunar/tc/qmq/producer/sender/RouterManager.java new file mode 100644 index 00000000..c8ff8420 --- /dev/null +++ b/qmq-client/src/main/java/qunar/tc/qmq/producer/sender/RouterManager.java @@ -0,0 +1,37 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.producer.sender; + +import qunar.tc.qmq.Message; +import qunar.tc.qmq.producer.QueueSender; + +public interface RouterManager { + + void init(String clientId); + + String name(); + + String registryOf(Message message); + + Connection routeOf(Message message); + + QueueSender getSender(); + + void validateMessage(Message message); + + void destroy(); +} diff --git a/qmq-client/src/main/java/qunar/tc/qmq/tracing/QmqMessageExtractAdapter.java b/qmq-client/src/main/java/qunar/tc/qmq/tracing/QmqMessageExtractAdapter.java new file mode 100644 index 00000000..dbb48d96 --- /dev/null +++ b/qmq-client/src/main/java/qunar/tc/qmq/tracing/QmqMessageExtractAdapter.java @@ -0,0 +1,50 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.tracing; + +import io.opentracing.propagation.TextMap; +import qunar.tc.qmq.Message; + +import java.util.HashMap; +import java.util.Iterator; +import java.util.Map; + +class QmqMessageExtractAdapter implements TextMap { + + private final Map map = new HashMap<>(); + + public QmqMessageExtractAdapter(Message message) { + Map attrs = message.getAttrs(); + for (Map.Entry entry : attrs.entrySet()) { + if (TraceUtil.isTraceKey(entry.getKey())) { + if (entry.getValue() == null) continue; + map.put(TraceUtil.extractKey(entry.getKey()), entry.getValue().toString()); + } + } + } + + @Override + public Iterator> iterator() { + return map.entrySet().iterator(); + } + + @Override + public void put(String key, String value) { + throw new UnsupportedOperationException( + "HeadersMapExtractAdapter should only be used with Tracer.extract()"); + } +} diff --git a/qmq-client/src/main/java/qunar/tc/qmq/tracing/QmqMessageInjectAdapter.java b/qmq-client/src/main/java/qunar/tc/qmq/tracing/QmqMessageInjectAdapter.java new file mode 100644 index 00000000..cfd567b2 --- /dev/null +++ b/qmq-client/src/main/java/qunar/tc/qmq/tracing/QmqMessageInjectAdapter.java @@ -0,0 +1,42 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.tracing; + +import io.opentracing.propagation.TextMap; +import qunar.tc.qmq.Message; + +import java.util.Iterator; +import java.util.Map; + +class QmqMessageInjectAdapter implements TextMap { + + private final Message message; + + public QmqMessageInjectAdapter(Message message) { + this.message = message; + } + + @Override + public Iterator> iterator() { + throw new UnsupportedOperationException("iterator should never be used with Tracer.inject()"); + } + + @Override + public void put(String key, String value) { + message.setProperty(TraceUtil.TRACE_PREFIX + key, value); + } +} diff --git a/qmq-client/src/main/java/qunar/tc/qmq/tracing/TraceUtil.java b/qmq-client/src/main/java/qunar/tc/qmq/tracing/TraceUtil.java new file mode 100644 index 00000000..267c4391 --- /dev/null +++ b/qmq-client/src/main/java/qunar/tc/qmq/tracing/TraceUtil.java @@ -0,0 +1,73 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.tracing; + +import io.opentracing.Scope; +import io.opentracing.SpanContext; +import io.opentracing.Tracer; +import io.opentracing.propagation.Format; +import io.opentracing.util.GlobalTracer; +import qunar.tc.qmq.Message; + +public class TraceUtil { + static final String TRACE_PREFIX = "qtrace_"; + + static String extractKey(String messageKey) { + return messageKey.substring(TRACE_PREFIX.length()); + } + + static boolean isTraceKey(String key) { + if (key == null) return false; + return key.startsWith(TRACE_PREFIX); + } + + public static void inject(Message message, Tracer tracer) { + Scope scope = tracer.scopeManager().active(); + if (scope == null) return; + tracer.inject(scope.span().context(), Format.Builtin.TEXT_MAP, new QmqMessageInjectAdapter(message)); + } + + public static SpanContext extract(Message message, Tracer tracer) { + return tracer.extract(Format.Builtin.TEXT_MAP, new QmqMessageExtractAdapter(message)); + } + + public static void setTag(String key, String value, Tracer tracer) { + if (tracer == null) { + tracer = GlobalTracer.get(); + } + Scope scope = tracer.scopeManager().active(); + if (scope == null) return; + scope.span().setTag(key, value); + } + + public static void setTag(String key, String value) { + setTag(key, value, null); + } + + public static void recordEvent(String event, Tracer tracer) { + if (tracer == null) { + tracer = GlobalTracer.get(); + } + Scope scope = tracer.scopeManager().active(); + if (scope == null) return; + scope.span().log(event); + } + + public static void recordEvent(String event) { + recordEvent(event, null); + } +} diff --git a/qmq-client/src/main/resources/META-INF/qmq-2.0.0.xsd b/qmq-client/src/main/resources/META-INF/qmq-2.0.0.xsd new file mode 100644 index 00000000..7686fb81 --- /dev/null +++ b/qmq-client/src/main/resources/META-INF/qmq-2.0.0.xsd @@ -0,0 +1,33 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/qmq-client/src/main/resources/META-INF/spring.handlers b/qmq-client/src/main/resources/META-INF/spring.handlers new file mode 100644 index 00000000..948cb447 --- /dev/null +++ b/qmq-client/src/main/resources/META-INF/spring.handlers @@ -0,0 +1 @@ +http\://www.qunar.com/schema/qmq=qunar.tc.qmq.consumer.annotation.QmqClientNamespaceHandler \ No newline at end of file diff --git a/qmq-client/src/main/resources/META-INF/spring.schemas b/qmq-client/src/main/resources/META-INF/spring.schemas new file mode 100644 index 00000000..62000e68 --- /dev/null +++ b/qmq-client/src/main/resources/META-INF/spring.schemas @@ -0,0 +1,2 @@ +http\://www.qunar.com/schema/qmq/qmq-2.0.0.xsd=META-INF/qmq-2.0.0.xsd +http\://www.qunar.com/schema/qmq/qmq.xsd=META-INF/qmq-2.0.0.xsd \ No newline at end of file diff --git a/qmq-common/pom.xml b/qmq-common/pom.xml new file mode 100644 index 00000000..4f79b3d7 --- /dev/null +++ b/qmq-common/pom.xml @@ -0,0 +1,49 @@ + + + 4.0.0 + + + qmq + qunar.tc + 4.0.30 + + + qmq-common + + + 1.7 + 1.7 + + + + + ${project.groupId} + qmq-api + + + org.slf4j + slf4j-api + + + com.google.guava + guava + + + com.fasterxml.jackson.core + jackson-annotations + + + + io.netty + netty-all + + + + + junit + junit + + + \ No newline at end of file diff --git a/qmq-common/src/main/java/qunar/tc/qmq/base/BaseMessage.java b/qmq-common/src/main/java/qunar/tc/qmq/base/BaseMessage.java new file mode 100644 index 00000000..11d21051 --- /dev/null +++ b/qmq-common/src/main/java/qunar/tc/qmq/base/BaseMessage.java @@ -0,0 +1,399 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ +package qunar.tc.qmq.base; + +import com.google.common.base.Preconditions; +import com.google.common.base.Strings; +import com.google.common.collect.Sets; +import qunar.tc.qmq.Message; +import qunar.tc.qmq.utils.RetrySubjectUtils; + +import java.io.Serializable; +import java.util.*; +import java.util.concurrent.CopyOnWriteArraySet; +import java.util.concurrent.TimeUnit; + +/** + * @author miao.yang susing@gmail.com + * @date 2012-12-26 + */ +public class BaseMessage implements Message, Serializable { + private static final long serialVersionUID = 303069262539600333L; + + private static final int MAX_MESSAGE_ID_LEN = 100; + + private static final int MAX_TAGS_COUNT = 10; + + String messageId; + + String subject; + + private final transient Set tags = new CopyOnWriteArraySet<>(); + + transient boolean isBigMessage = false; + + public enum keys { + qmq_createTime, + qmq_expireTime, + qmq_consumerGroupName, + qmq_scheduleReceiveTime, + qmq_times, + qmq_maxRetryNum, + qmq_appCode, + qmq_pullOffset, + qmq_corruptData + } + + private static final Set keyNames = Sets.newHashSet(); + + static { + for (keys key : keys.values()) + keyNames.add(key.name()); + } + + HashMap attrs = new HashMap<>(); + + public BaseMessage() { + } + + public BaseMessage(String messageId, String subject) { + Preconditions.checkArgument(!Strings.isNullOrEmpty(messageId), "message id should not empty"); + Preconditions.checkArgument(!Strings.isNullOrEmpty(subject), "message subject should not empty"); + Preconditions.checkArgument(messageId.length() <= MAX_MESSAGE_ID_LEN, "messageId长度不能超过" + MAX_MESSAGE_ID_LEN + "个字符"); + if (RetrySubjectUtils.isRealSubject(subject)) { + Preconditions.checkArgument(subject.length() <= MAX_MESSAGE_ID_LEN, "subject长度不能超过" + MAX_MESSAGE_ID_LEN + "个字符"); + } + + this.messageId = messageId; + this.subject = subject; + long time = System.currentTimeMillis(); + setProperty(keys.qmq_createTime, time); + } + + public BaseMessage(BaseMessage message) { + this(message.getMessageId(), message.getSubject()); + this.tags.addAll(message.getTags()); + this.attrs = new HashMap<>(message.attrs); + } + + public Map getAttrs() { + return Collections.unmodifiableMap(attrs); + } + + @Deprecated + public void setAttrs(HashMap attrs) { + this.attrs = attrs; + } + + @Override + public String getMessageId() { + return messageId; + } + + @Override + public String getSubject() { + return subject; + } + + public void setMessageId(String messageId) { + this.messageId = messageId; + } + + public void setSubject(String subject) { + this.subject = subject; + } + + @Override + public Date getCreatedTime() { + return getDateProperty(keys.qmq_createTime.name()); + } + + public void setProperty(keys key, boolean value) { + attrs.put(key.name(), Boolean.valueOf(value)); + } + + public void setProperty(keys key, String value) { + attrs.put(key.name(), value); + } + + public void setProperty(keys key, int value) { + attrs.put(key.name(), value); + } + + public void setProperty(keys key, long value) { + attrs.put(key.name(), value); + } + + public void setProperty(keys key, Date value) { + attrs.put(key.name(), value.getTime()); + } + + /** + * 为了类型属性的稳定此方法一定不能暴漏成public. + */ + private void setObjectProperty(String name, Object value) { + if (keyNames.contains(name)) + throw new IllegalArgumentException("property name [" + name + "] is protected. "); + attrs.put(name, value); + } + + @Override + public void setProperty(String name, boolean value) { + setObjectProperty(name, value); + } + + @Override + public void setProperty(String name, Boolean value) { + if (value == null) return; + setObjectProperty(name, value); + } + + @Override + public void setProperty(String name, int value) { + setObjectProperty(name, value); + } + + @Override + public void setProperty(String name, Integer value) { + if (value == null) return; + setObjectProperty(name, value); + } + + @Override + public void setProperty(String name, long value) { + setObjectProperty(name, value); + } + + @Override + public void setProperty(String name, Long value) { + if (value == null) return; + setObjectProperty(name, value); + } + + @Override + public void setProperty(String name, float value) { + setObjectProperty(name, value); + } + + @Override + public void setProperty(String name, Float value) { + if (value == null) return; + setObjectProperty(name, value); + } + + @Override + public void setProperty(String name, double value) { + setObjectProperty(name, value); + } + + @Override + public void setProperty(String name, Double value) { + if (value == null) return; + setObjectProperty(name, value); + } + + @Override + public void setProperty(String name, Date value) { + if (value == null) return; + setObjectProperty(name, value.getTime()); + } + + @Override + public void setProperty(String name, String value) { + if (value == null) return; + setObjectProperty(name, value); + } + + @Override + public void setLargeString(String name, String value) { + LargeStringUtil.setLargeString(this, name, value); + } + + @Override + public String getStringProperty(String name) { + return valueOfString(attrs.get(name)); + } + + @Override + public boolean getBooleanProperty(String name) { + Object v = attrs.get(name); + if (v == null) + return false; + return Boolean.valueOf(v.toString()); + } + + @Override + public Date getDateProperty(String name) { + Object o = attrs.get(name); + if (o == null) + return null; + Long v = Long.valueOf(o.toString()); + return new Date(v); + } + + @Override + public int getIntProperty(String name) { + Object o = attrs.get(name); + if (o == null) + return 0; + return Integer.valueOf(o.toString()); + } + + @Override + public long getLongProperty(String name) { + Object o = attrs.get(name); + if (o == null) + return 0; + return Long.valueOf(o.toString()); + } + + @Override + public float getFloatProperty(String name) { + Object o = attrs.get(name); + if (o == null) + return 0; + return Float.valueOf(o.toString()); + } + + @Override + public double getDoubleProperty(String name) { + Object o = attrs.get(name); + if (o == null) + return 0; + return Double.valueOf(o.toString()); + } + + @Override + public String getLargeString(String name) { + return LargeStringUtil.getLargeString(this, name); + } + + private static String valueOfString(Object str) { + return str == null ? null : str.toString(); + } + + public Object getProperty(keys key) { + return attrs.get(key.name()); + } + + public String getStringProperty(keys key) { + return getStringProperty(key.name()); + } + + public void removeProperty(keys key) { + attrs.remove(key.name()); + } + + @Override + public Message addTag(String tag) { + Preconditions.checkArgument(!Strings.isNullOrEmpty(tag), "a tag can not be null or empty"); + Preconditions.checkArgument(tag.length() <= Short.MAX_VALUE, "the length of a tag mush be smaller than Short.MAX_VALUE"); + if (tags.size() >= MAX_TAGS_COUNT) { + throw new IllegalArgumentException("the size of tags cannot be more than MAX_TAGS_COUNT(" + MAX_TAGS_COUNT + ")"); + } + tags.add(tag); + return this; + } + + @Override + public Set getTags() { + return Collections.unmodifiableSet(tags); + } + + @Override + public void autoAck(boolean auto) { + throw new UnsupportedOperationException("请在consumer端设置auto ack"); + } + + @Override + public void ack(long elapsed, Throwable e) { + ack(elapsed, e, null); + } + + @Override + public void ack(long elapsed, Throwable e, Map attachment) { + throw new UnsupportedOperationException("BaseMessage does not support this method"); + } + + public void setExpiredTime(long time) { + setProperty(keys.qmq_expireTime, time); + } + + public void setExpiredDelay(long timeDelay, TimeUnit timeUnit) { + setExpiredTime(System.currentTimeMillis() + timeUnit.toMillis(timeDelay)); + } + + @Override + public void setDelayTime(Date date) { + Preconditions.checkNotNull(date, "消息定时接收时间不能为空"); + long time = date.getTime(); + Preconditions.checkArgument(time > System.currentTimeMillis(), "消息定时接收时间不能为过去时"); + setDelay(time); + } + + // WARNING setProperty(String + // name,...)这个版本的方法里面会对name进行检查,如果这个name在keys集合(qmq内部使用) + // 中则会抛出异常,这是为了防止业务使用到这些内部保留关键字。 + // 所以qmq内部使用的属性都应该使用setProperty(keys key,...)这个版本。 + private void setDelay(long time) { + setProperty(keys.qmq_scheduleReceiveTime, time); + } + + @Override + public void setDelayTime(long delayTime, TimeUnit timeUnit) { + Preconditions.checkNotNull(timeUnit, "消息延迟接收时间单位不能为空"); + Preconditions.checkArgument(delayTime >= 0, "消息延迟接收时间不能为过去时"); + + long sendTime = System.currentTimeMillis() + timeUnit.toMillis(delayTime); + setDelay(sendTime); + } + + @Override + public Date getScheduleReceiveTime() { + return getDateProperty(keys.qmq_scheduleReceiveTime.name()); + } + + @Override + public int times() { + Object o = getProperty(keys.qmq_times); + if (o == null) return 1; + return Integer.valueOf(o.toString()); + } + + @Override + public void setMaxRetryNum(int maxRetryNum) { + setProperty(keys.qmq_maxRetryNum, maxRetryNum); + } + + @Override + public int getMaxRetryNum() { + String value = getStringProperty(keys.qmq_maxRetryNum); + if (Strings.isNullOrEmpty(value)) { + return -1; + } + try { + return Integer.parseInt(value); + } catch (Exception e) { + return -1; + } + } + + @Override + public int localRetries() { + throw new UnsupportedOperationException("本地重试,只有消费端才支持"); + } + +} diff --git a/qmq-common/src/main/java/qunar/tc/qmq/base/ClientRequestType.java b/qmq-common/src/main/java/qunar/tc/qmq/base/ClientRequestType.java new file mode 100644 index 00000000..59596dfd --- /dev/null +++ b/qmq-common/src/main/java/qunar/tc/qmq/base/ClientRequestType.java @@ -0,0 +1,36 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.base; + +/** + * @author yunfeng.yang + * @since 2017/10/16 + */ +public enum ClientRequestType { + ONLINE(1), HEARTBEAT(2); + + private int code; + + ClientRequestType(int code) { + this.code = code; + } + + public int getCode() { + return code; + } +} + diff --git a/qmq-common/src/main/java/qunar/tc/qmq/base/LargeStringUtil.java b/qmq-common/src/main/java/qunar/tc/qmq/base/LargeStringUtil.java new file mode 100644 index 00000000..4b4f515e --- /dev/null +++ b/qmq-common/src/main/java/qunar/tc/qmq/base/LargeStringUtil.java @@ -0,0 +1,64 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.base; + +/** + * Created by zhaohui.yu + * 5/4/18 + */ +class LargeStringUtil { + private static final int _32K = (32 * 1024) / 4; + + static void setLargeString(BaseMessage msg, String key, String data) { + int len = data.length(); + if (len <= _32K) { + msg.setProperty(key, data); + return; + } + + msg.isBigMessage = true; + int partIdx = 0; + for (int remain = len; remain > 0; remain -= _32K) { + final int beginIdx = partIdx * _32K; + final int endIdx = beginIdx + Math.min(_32K, remain); + final String part = data.substring(beginIdx, endIdx); + msg.setProperty(buildPartKey(key, partIdx), part); + partIdx += 1; + } + } + + static String getLargeString(BaseMessage msg, String key) { + String small = msg.getStringProperty(key); + if (small != null) return small; + + StringBuilder result = new StringBuilder(); + int partIdx = 0; + while (true) { + String part = msg.getStringProperty(buildPartKey(key, partIdx)); + if (part == null) { + break; + } + partIdx += 1; + result.append(part); + } + return result.toString(); + } + + private static String buildPartKey(String key, int idx) { + return key + "#part" + idx; + } +} diff --git a/qmq-common/src/main/java/qunar/tc/qmq/base/MessageHeader.java b/qmq-common/src/main/java/qunar/tc/qmq/base/MessageHeader.java new file mode 100644 index 00000000..9888df92 --- /dev/null +++ b/qmq-common/src/main/java/qunar/tc/qmq/base/MessageHeader.java @@ -0,0 +1,90 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.base; + +import java.util.HashSet; +import java.util.Set; + +/** + * @author yiqun.fan create on 17-7-4. + */ +public class MessageHeader { + private long bodyCrc; + private byte flag; + private long createTime; + private long expireTime; + private String subject; + private String messageId; + + private Set tags = new HashSet<>(); + + public long getBodyCrc() { + return bodyCrc; + } + + public void setBodyCrc(long bodyCrc) { + this.bodyCrc = bodyCrc; + } + + public byte getFlag() { + return flag; + } + + public void setFlag(byte flag) { + this.flag = flag; + } + + public long getCreateTime() { + return createTime; + } + + public void setCreateTime(long createTime) { + this.createTime = createTime; + } + + public long getExpireTime() { + return expireTime; + } + + public void setExpireTime(long expireTime) { + this.expireTime = expireTime; + } + + public String getSubject() { + return subject; + } + + public void setSubject(String subject) { + this.subject = subject; + } + + public String getMessageId() { + return messageId; + } + + public void setMessageId(String messageId) { + this.messageId = messageId; + } + + public Set getTags() { + return tags; + } + + public void setTags(Set tags) { + this.tags = tags; + } +} diff --git a/qmq-common/src/main/java/qunar/tc/qmq/base/OnOfflineState.java b/qmq-common/src/main/java/qunar/tc/qmq/base/OnOfflineState.java new file mode 100644 index 00000000..07a6d628 --- /dev/null +++ b/qmq-common/src/main/java/qunar/tc/qmq/base/OnOfflineState.java @@ -0,0 +1,44 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.base; + +/** + * yiqun.fan@qunar.com 2018/3/2 + */ +public enum OnOfflineState { + + ONLINE(0), OFFLINE(1); + + private final int code; + + OnOfflineState(int code) { + this.code = code; + } + + public int code() { + return code; + } + + @Override + public String toString() { + return code == 0 ? "ON" : "OFF"; + } + + public static OnOfflineState fromCode(int code) { + return code == 0 ? ONLINE : OFFLINE; + } +} diff --git a/qmq-common/src/main/java/qunar/tc/qmq/base/RawMessage.java b/qmq-common/src/main/java/qunar/tc/qmq/base/RawMessage.java new file mode 100644 index 00000000..889fa839 --- /dev/null +++ b/qmq-common/src/main/java/qunar/tc/qmq/base/RawMessage.java @@ -0,0 +1,54 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.base; + +import io.netty.buffer.ByteBuf; + +/** + * @author yiqun.fan create on 17-7-4. + */ +public class RawMessage { + private final MessageHeader header; + private final ByteBuf body; + private final int bodySize; + + public RawMessage(MessageHeader header, ByteBuf body, int size) { + this.header = header; + this.body = body; + this.bodySize = size; + } + + public MessageHeader getHeader() { + return header; + } + + public ByteBuf getBody() { + return body; + } + + public int getBodySize() { + return bodySize; + } + + public void setSubject(String subject) { + header.setSubject(subject); + } + + public boolean isHigh() { + return (header.getFlag() & 1) == 0; + } +} diff --git a/qmq-common/src/main/java/qunar/tc/qmq/batch/BatchExecutor.java b/qmq-common/src/main/java/qunar/tc/qmq/batch/BatchExecutor.java new file mode 100644 index 00000000..a83a01e0 --- /dev/null +++ b/qmq-common/src/main/java/qunar/tc/qmq/batch/BatchExecutor.java @@ -0,0 +1,113 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.batch; + +import com.google.common.base.Preconditions; +import com.google.common.collect.Lists; +import qunar.tc.qmq.concurrent.NamedThreadFactory; + +import javax.annotation.PostConstruct; +import javax.annotation.PreDestroy; +import java.util.List; +import java.util.concurrent.*; + +/** + * User: zhaohuiyu Date: 6/4/13 Time: 5:21 PM + */ +public class BatchExecutor implements Runnable { + private final String name; + private final int batchSize; + private final Processor processor; + + private static final int DEFAULT_QUEUE_SIZE = 1000; + private static final int DEFAULT_PROCESS_THREADS = Runtime.getRuntime().availableProcessors() + 1; + + private int queueSize = DEFAULT_QUEUE_SIZE; + private int threads; + + private BlockingQueue queue; + private ThreadPoolExecutor executor; + + public BatchExecutor(String name, int batchSize, Processor processor) { + this(name, batchSize, processor, DEFAULT_PROCESS_THREADS); + } + + public BatchExecutor(String name, int batchSize, Processor processor, int threads) { + Preconditions.checkNotNull(processor); + + this.name = name; + this.batchSize = batchSize; + this.processor = processor; + this.threads = threads; + } + + @PostConstruct + public void init() { + this.queue = new LinkedBlockingQueue<>(this.queueSize); + if (this.executor == null) { + this.executor = new ThreadPoolExecutor(1, threads, 1L, TimeUnit.MINUTES, + new ArrayBlockingQueue(1), new NamedThreadFactory("batch-" + name + "-task", true)); + executor.setRejectedExecutionHandler(new ThreadPoolExecutor.DiscardPolicy()); + } + } + + public boolean addItem(Item item) { + boolean offer = this.queue.offer(item); + if (offer) { + this.executor.execute(this); + } + return offer; + } + + public boolean addItem(Item item, long timeout, TimeUnit unit) throws InterruptedException { + boolean offer = this.queue.offer(item, timeout, unit); + if (offer) { + this.executor.execute(this); + } + return offer; + } + + @Override + public void run() { + while (!this.queue.isEmpty()) { + List list = Lists.newArrayListWithCapacity(batchSize); + int size = this.queue.drainTo(list, batchSize); + if (size > 0) { + this.processor.process(list); + } + } + } + + public void setQueueSize(int queueSize) { + this.queueSize = queueSize; + } + + public void setThreads(int threads) { + this.threads = threads; + } + + public void setExecutor(ThreadPoolExecutor executor) { + this.executor = executor; + } + + @PreDestroy + public void destroy() { + if (executor != null) { + executor.shutdown(); + } + } +} diff --git a/qmq-common/src/main/java/qunar/tc/qmq/batch/Processor.java b/qmq-common/src/main/java/qunar/tc/qmq/batch/Processor.java new file mode 100644 index 00000000..b78d5a82 --- /dev/null +++ b/qmq-common/src/main/java/qunar/tc/qmq/batch/Processor.java @@ -0,0 +1,28 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.batch; + +import java.util.List; + +/** + * User: zhaohuiyu + * Date: 6/4/13 + * Time: 5:21 PM + */ +public interface Processor { + void process(List items); +} diff --git a/qmq-common/src/main/java/qunar/tc/qmq/common/ClientType.java b/qmq-common/src/main/java/qunar/tc/qmq/common/ClientType.java new file mode 100644 index 00000000..2f8b5717 --- /dev/null +++ b/qmq-common/src/main/java/qunar/tc/qmq/common/ClientType.java @@ -0,0 +1,65 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.common; + +import com.google.common.collect.ImmutableMap; + +import java.util.HashMap; +import java.util.Map; + +/** + * @author yiqun.fan create on 17-8-22. + */ +public enum ClientType { + PRODUCER(1), + CONSUMER(2), + OTHER(3), + DELAY_PRODUCER(4); + + private static final ImmutableMap INSTANCES; + + static { + final Map result = new HashMap<>(); + for (final ClientType type : values()) { + result.put(type.getCode(), type); + } + INSTANCES = ImmutableMap.copyOf(result); + } + + private int code; + + ClientType(int code) { + this.code = code; + } + + public static ClientType of(final int code) { + ClientType type = INSTANCES.get(code); + return type == null ? OTHER : type; + } + + public int getCode() { + return code; + } + + public boolean isProducer() { + return code == PRODUCER.code || code == DELAY_PRODUCER.code; + } + + public boolean isConsumer() { + return code == CONSUMER.code; + } +} diff --git a/qmq-common/src/main/java/qunar/tc/qmq/common/Disposable.java b/qmq-common/src/main/java/qunar/tc/qmq/common/Disposable.java new file mode 100644 index 00000000..5811dbc2 --- /dev/null +++ b/qmq-common/src/main/java/qunar/tc/qmq/common/Disposable.java @@ -0,0 +1,26 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.common; + +/** + * User: zhaohuiyu + * Date: 7/24/13 + * Time: 3:17 PM + */ +public interface Disposable { + void destroy(); +} diff --git a/qmq-common/src/main/java/qunar/tc/qmq/concurrent/NamedThreadFactory.java b/qmq-common/src/main/java/qunar/tc/qmq/concurrent/NamedThreadFactory.java new file mode 100644 index 00000000..287f0639 --- /dev/null +++ b/qmq-common/src/main/java/qunar/tc/qmq/concurrent/NamedThreadFactory.java @@ -0,0 +1,49 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.concurrent; + +import java.util.concurrent.ThreadFactory; +import java.util.concurrent.atomic.AtomicInteger; + +public class NamedThreadFactory implements ThreadFactory { + + private final AtomicInteger mThreadNum = new AtomicInteger(1); + + private final String mPrefix; + + private final boolean mDaemo; + + private final ThreadGroup mGroup; + + public NamedThreadFactory(String prefix) { + this(prefix, true); + } + + public NamedThreadFactory(String prefix, boolean daemo) { + mPrefix = prefix + "-thread-"; + mDaemo = daemo; + SecurityManager s = System.getSecurityManager(); + mGroup = (s == null) ? Thread.currentThread().getThreadGroup() : s.getThreadGroup(); + } + + public Thread newThread(Runnable runnable) { + String name = mPrefix + mThreadNum.getAndIncrement(); + Thread ret = new Thread(mGroup, runnable, name, 0); + ret.setDaemon(mDaemo); + return ret; + } +} diff --git a/qmq-common/src/main/java/qunar/tc/qmq/configuration/DynamicConfig.java b/qmq-common/src/main/java/qunar/tc/qmq/configuration/DynamicConfig.java new file mode 100644 index 00000000..9c84ef0d --- /dev/null +++ b/qmq-common/src/main/java/qunar/tc/qmq/configuration/DynamicConfig.java @@ -0,0 +1,48 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.configuration; + +import java.util.Map; + +/** + * User: zhaohuiyu Date: 12/24/12 Time: 4:12 PM + */ +public interface DynamicConfig { + void addListener(Listener listener); + + String getString(String name); + + String getString(String name, String defaultValue); + + int getInt(String name); + + int getInt(String name, int defaultValue); + + long getLong(String name); + + long getLong(String name, long defaultValue); + + double getDouble(String name); + + double getDouble(String name, double defaultValue); + + boolean getBoolean(String name, boolean defaultValue); + + boolean exist(String name); + + Map asMap(); +} diff --git a/qmq-common/src/main/java/qunar/tc/qmq/configuration/DynamicConfigFactory.java b/qmq-common/src/main/java/qunar/tc/qmq/configuration/DynamicConfigFactory.java new file mode 100644 index 00000000..8ba69040 --- /dev/null +++ b/qmq-common/src/main/java/qunar/tc/qmq/configuration/DynamicConfigFactory.java @@ -0,0 +1,25 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.configuration; + +/** + * @author keli.wang + * @since 2018-11-23 + */ +public interface DynamicConfigFactory { + DynamicConfig create(String name, boolean failOnNotExist); +} diff --git a/qmq-common/src/main/java/qunar/tc/qmq/configuration/DynamicConfigLoader.java b/qmq-common/src/main/java/qunar/tc/qmq/configuration/DynamicConfigLoader.java new file mode 100644 index 00000000..eca74a17 --- /dev/null +++ b/qmq-common/src/main/java/qunar/tc/qmq/configuration/DynamicConfigLoader.java @@ -0,0 +1,50 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.configuration; + +import java.util.ServiceLoader; + +/** + * @author keli.wang + * @since 2018-11-23 + */ +public final class DynamicConfigLoader { + // TODO(keli.wang): can we set this using config? + private static final DynamicConfigFactory FACTORY; + + static { + ServiceLoader factories = ServiceLoader.load(DynamicConfigFactory.class); + DynamicConfigFactory instance = null; + for (DynamicConfigFactory factory : factories) { + instance = factory; + break; + } + + FACTORY = instance; + } + + private DynamicConfigLoader() { + } + + public static DynamicConfig load(final String name) { + return load(name, true); + } + + public static DynamicConfig load(final String name, final boolean failOnNotExist) { + return FACTORY.create(name, failOnNotExist); + } +} diff --git a/qmq-common/src/main/java/qunar/tc/qmq/configuration/Listener.java b/qmq-common/src/main/java/qunar/tc/qmq/configuration/Listener.java new file mode 100644 index 00000000..2617a3a1 --- /dev/null +++ b/qmq-common/src/main/java/qunar/tc/qmq/configuration/Listener.java @@ -0,0 +1,25 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.configuration; + +/** + * @author keli.wang + * @since 2018-11-27 + */ +public interface Listener { + void onLoad(DynamicConfig config); +} diff --git a/qmq-common/src/main/java/qunar/tc/qmq/configuration/local/ConfigWatcher.java b/qmq-common/src/main/java/qunar/tc/qmq/configuration/local/ConfigWatcher.java new file mode 100644 index 00000000..ce26d315 --- /dev/null +++ b/qmq-common/src/main/java/qunar/tc/qmq/configuration/local/ConfigWatcher.java @@ -0,0 +1,101 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.configuration.local; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import qunar.tc.qmq.concurrent.NamedThreadFactory; + +import java.util.concurrent.CopyOnWriteArrayList; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; + +/** + * @author keli.wang + * @since 2018-11-27 + */ +class ConfigWatcher { + private static final Logger LOG = LoggerFactory.getLogger(ConfigWatcher.class); + + private final CopyOnWriteArrayList watches; + private final ScheduledExecutorService watcherExecutor; + + ConfigWatcher() { + this.watches = new CopyOnWriteArrayList<>(); + this.watcherExecutor = Executors.newSingleThreadScheduledExecutor(new NamedThreadFactory("local-config-watcher")); + + start(); + } + + private void start() { + watcherExecutor.scheduleWithFixedDelay(new Runnable() { + @Override + public void run() { + checkAllWatches(); + } + }, 10, 10, TimeUnit.SECONDS); + } + + private void checkAllWatches() { + for (Watch watch : watches) { + try { + checkWatch(watch); + } catch (Exception e) { + LOG.error("check config failed. config: {}", watch.getConfig(), e); + } + } + } + + private void checkWatch(final Watch watch) { + final LocalDynamicConfig config = watch.getConfig(); + final long lastModified = config.getLastModified(); + if (lastModified == watch.getLastModified()) { + return; + } + + watch.setLastModified(lastModified); + config.onConfigModified(); + } + + void addWatch(final LocalDynamicConfig config) { + final Watch watch = new Watch(config); + watch.setLastModified(config.getLastModified()); + watches.add(watch); + } + + private static final class Watch { + private final LocalDynamicConfig config; + private volatile long lastModified; + + private Watch(final LocalDynamicConfig config) { + this.config = config; + } + + public LocalDynamicConfig getConfig() { + return config; + } + + long getLastModified() { + return lastModified; + } + + void setLastModified(final long lastModified) { + this.lastModified = lastModified; + } + } +} diff --git a/qmq-common/src/main/java/qunar/tc/qmq/configuration/local/LocalDynamicConfig.java b/qmq-common/src/main/java/qunar/tc/qmq/configuration/local/LocalDynamicConfig.java new file mode 100644 index 00000000..0bf52273 --- /dev/null +++ b/qmq-common/src/main/java/qunar/tc/qmq/configuration/local/LocalDynamicConfig.java @@ -0,0 +1,238 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.configuration.local; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import qunar.tc.qmq.configuration.DynamicConfig; +import qunar.tc.qmq.configuration.Listener; + +import java.io.*; +import java.net.URISyntaxException; +import java.net.URL; +import java.nio.file.Paths; +import java.util.*; +import java.util.concurrent.CopyOnWriteArrayList; + +/** + * @author keli.wang + * @since 2018-11-23 + */ +public class LocalDynamicConfig implements DynamicConfig { + private static final Logger LOG = LoggerFactory.getLogger(LocalDynamicConfig.class); + + private final String name; + private final CopyOnWriteArrayList listeners; + private volatile File file; + private volatile boolean loaded = false; + private volatile Map config; + + LocalDynamicConfig(String name, boolean failOnNotExist) { + this.name = name; + this.listeners = new CopyOnWriteArrayList<>(); + this.file = getFileByName(name); + + if (failOnNotExist && (file == null || !file.exists())) { + throw new RuntimeException("cannot find config file " + name); + } + } + + private File getFileByName(final String name) { + try { + final URL res = this.getClass().getClassLoader().getResource(name); + if (res == null) { + return null; + } + return Paths.get(res.toURI()).toFile(); + } catch (URISyntaxException e) { + throw new RuntimeException("load config file failed", e); + } + } + + long getLastModified() { + if (file == null) { + file = getFileByName(name); + } + + if (file == null) { + return 0; + } else { + return file.lastModified(); + } + } + + synchronized void onConfigModified() { + if (file == null) { + return; + } + + loadConfig(); + executeListeners(); + loaded = true; + } + + private void loadConfig() { + try { + final Properties p = new Properties(); + try (Reader reader = new BufferedReader(new FileReader(file))) { + p.load(reader); + } + final Map map = new LinkedHashMap<>(p.size()); + for (String key : p.stringPropertyNames()) { + map.put(key, tryTrim(p.getProperty(key))); + } + + config = Collections.unmodifiableMap(map); + } catch (IOException e) { + LOG.error("load local config failed. config: {}", file.getAbsolutePath(), e); + } + } + + private String tryTrim(String data) { + if (data == null) { + return null; + } else { + return data.trim(); + } + } + + private void executeListeners() { + for (Listener listener : listeners) { + executeListener(listener); + } + } + + @Override + public void addListener(Listener listener) { + if (loaded) { + executeListener(listener); + } + listeners.add(listener); + } + + private void executeListener(Listener listener) { + try { + listener.onLoad(this); + } catch (Throwable e) { + LOG.error("trigger config listener failed. config: {}", name, e); + } + } + + @Override + public String getString(String name) { + return getValueWithCheck(name); + } + + @Override + public String getString(String name, String defaultValue) { + String value = getValue(name); + if (isBlank(value)) + return defaultValue; + return value; + } + + @Override + public int getInt(String name) { + return Integer.valueOf(getValueWithCheck(name)); + } + + @Override + public int getInt(String name, int defaultValue) { + String value = getValue(name); + if (isBlank(value)) + return defaultValue; + return Integer.valueOf(value); + } + + @Override + public long getLong(String name) { + return Long.valueOf(getValueWithCheck(name)); + } + + @Override + public long getLong(String name, long defaultValue) { + String value = getValue(name); + if (isBlank(value)) + return defaultValue; + return Long.valueOf(value); + } + + @Override + public double getDouble(final String name) { + return Double.valueOf(getValueWithCheck(name)); + } + + @Override + public double getDouble(final String name, final double defaultValue) { + String value = getValue(name); + if (isBlank(value)) + return defaultValue; + return Double.valueOf(value); + } + + @Override + public boolean getBoolean(String name, boolean defaultValue) { + String value = getValue(name); + if (isBlank(value)) + return defaultValue; + return Boolean.valueOf(value); + } + + + private String getValueWithCheck(String name) { + String value = getValue(name); + if (isBlank(value)) { + throw new RuntimeException("配置项: " + name + " 值为空"); + } else { + return value; + } + } + + private String getValue(String name) { + return config.get(name); + } + + private boolean isBlank(final String s) { + if (s == null || s.isEmpty()) { + return true; + } + + for (int i = 0; i < s.length(); i++) { + if (!Character.isWhitespace(s.charAt(i))) { + return false; + } + } + return true; + } + + @Override + public boolean exist(String name) { + return config.containsKey(name); + } + + @Override + public Map asMap() { + return new HashMap<>(config); + } + + @Override + public String toString() { + return "LocalDynamicConfig{" + + "name='" + name + '\'' + + '}'; + } +} diff --git a/qmq-common/src/main/java/qunar/tc/qmq/configuration/local/LocalDynamicConfigFactory.java b/qmq-common/src/main/java/qunar/tc/qmq/configuration/local/LocalDynamicConfigFactory.java new file mode 100644 index 00000000..7d7b580a --- /dev/null +++ b/qmq-common/src/main/java/qunar/tc/qmq/configuration/local/LocalDynamicConfigFactory.java @@ -0,0 +1,51 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.configuration.local; + +import qunar.tc.qmq.configuration.DynamicConfig; +import qunar.tc.qmq.configuration.DynamicConfigFactory; + +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; + +/** + * @author keli.wang + * @since 2018-11-27 + */ +public class LocalDynamicConfigFactory implements DynamicConfigFactory { + private final ConfigWatcher watcher = new ConfigWatcher(); + private final ConcurrentMap configs = new ConcurrentHashMap<>(); + + @Override + public DynamicConfig create(final String name, final boolean failOnNotExist) { + if (configs.containsKey(name)) { + return configs.get(name); + } + + return doCreate(name, failOnNotExist); + } + + private LocalDynamicConfig doCreate(final String name, final boolean failOnNotExist) { + final LocalDynamicConfig prev = configs.putIfAbsent(name, new LocalDynamicConfig(name, failOnNotExist)); + final LocalDynamicConfig config = configs.get(name); + if (prev == null) { + watcher.addWatch(config); + config.onConfigModified(); + } + return config; + } +} diff --git a/qmq-common/src/main/java/qunar/tc/qmq/meta/BrokerCluster.java b/qmq-common/src/main/java/qunar/tc/qmq/meta/BrokerCluster.java new file mode 100644 index 00000000..07cb34e8 --- /dev/null +++ b/qmq-common/src/main/java/qunar/tc/qmq/meta/BrokerCluster.java @@ -0,0 +1,39 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.meta; + +import java.util.List; + +/** + * @author yunfeng.yang + * @since 2017/8/28 + */ +public class BrokerCluster { + private List brokerGroups; + + public BrokerCluster(List brokerGroups) { + this.brokerGroups = brokerGroups; + } + + public List getBrokerGroups() { + return brokerGroups; + } + + public void setBrokerGroups(List brokerGroups) { + this.brokerGroups = brokerGroups; + } +} diff --git a/qmq-common/src/main/java/qunar/tc/qmq/meta/BrokerGroup.java b/qmq-common/src/main/java/qunar/tc/qmq/meta/BrokerGroup.java new file mode 100644 index 00000000..18569e7c --- /dev/null +++ b/qmq-common/src/main/java/qunar/tc/qmq/meta/BrokerGroup.java @@ -0,0 +1,116 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.meta; + +import java.util.List; +import java.util.Objects; + +/** + * @author yunfeng.yang + * @since 2017/8/28 + */ +public class BrokerGroup { + private String groupName; + private String master; + private List slaves; + private long updateTime; + private BrokerState brokerState; + private String tag; + private BrokerGroupKind kind; + + public String getGroupName() { + return groupName; + } + + public void setGroupName(String groupName) { + this.groupName = groupName; + } + + public String getMaster() { + return master; + } + + public void setMaster(String master) { + this.master = master; + } + + public BrokerState getBrokerState() { + return brokerState; + } + + public void setBrokerState(BrokerState brokerState) { + this.brokerState = brokerState; + } + + public List getSlaves() { + return slaves; + } + + public void setSlaves(List slaves) { + this.slaves = slaves; + } + + public long getUpdateTime() { + return updateTime; + } + + public void setUpdateTime(long updateTime) { + this.updateTime = updateTime; + } + + public String getTag() { + return tag; + } + + public void setTag(String tag) { + this.tag = tag; + } + + public BrokerGroupKind getKind() { + return kind; + } + + public void setKind(final BrokerGroupKind kind) { + this.kind = kind; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + BrokerGroup group = (BrokerGroup) o; + return Objects.equals(groupName, group.groupName); + } + + @Override + public int hashCode() { + + return Objects.hash(groupName); + } + + @Override + public String toString() { + return "BrokerGroup{" + + "groupName='" + groupName + '\'' + + ", master='" + master + '\'' + + ", slaves=" + slaves + + ", updateTime=" + updateTime + + ", brokerState=" + brokerState + + ", tag='" + tag + '\'' + + '}'; + } +} diff --git a/qmq-common/src/main/java/qunar/tc/qmq/meta/BrokerGroupKind.java b/qmq-common/src/main/java/qunar/tc/qmq/meta/BrokerGroupKind.java new file mode 100644 index 00000000..27612be2 --- /dev/null +++ b/qmq-common/src/main/java/qunar/tc/qmq/meta/BrokerGroupKind.java @@ -0,0 +1,45 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.meta; + +/** + * @author keli.wang + * @since 2018-12-05 + */ +public enum BrokerGroupKind { + NORMAL(1), + DELAY(2); + + private final int code; + + BrokerGroupKind(final int code) { + this.code = code; + } + + public static BrokerGroupKind fromCode(final int code) { + for (final BrokerGroupKind kind : values()) { + if (kind.getCode() == code) { + return kind; + } + } + + throw new RuntimeException("unknown broker group kind code " + code); + } + + public int getCode() { + return code; + }} diff --git a/qmq-common/src/main/java/qunar/tc/qmq/meta/BrokerState.java b/qmq-common/src/main/java/qunar/tc/qmq/meta/BrokerState.java new file mode 100644 index 00000000..5db635e8 --- /dev/null +++ b/qmq-common/src/main/java/qunar/tc/qmq/meta/BrokerState.java @@ -0,0 +1,52 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.meta; + +/** + * @author yunfeng.yang + * @since 2017/8/28 + */ +public enum BrokerState { + RW(1), R(2), W(3), NRW(4); + + private final int code; + + BrokerState(int code) { + this.code = code; + } + + public static BrokerState codeOf(int brokerState) { + for (BrokerState value : BrokerState.values()) { + if (value.getCode() == brokerState) { + return value; + } + } + return null; + } + + public int getCode() { + return code; + } + + public boolean canRead() { + return code == RW.code || code == R.code; + } + + public boolean canWrite() { + return code == RW.code || code == W.code; + } +} diff --git a/qmq-common/src/main/java/qunar/tc/qmq/meta/MetaServerLocator.java b/qmq-common/src/main/java/qunar/tc/qmq/meta/MetaServerLocator.java new file mode 100644 index 00000000..f3a4f459 --- /dev/null +++ b/qmq-common/src/main/java/qunar/tc/qmq/meta/MetaServerLocator.java @@ -0,0 +1,101 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.meta; + +import com.google.common.base.Optional; +import com.google.common.io.CharStreams; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import qunar.tc.qmq.utils.NetworkUtils; + +import java.io.Closeable; +import java.io.IOException; +import java.io.InputStreamReader; +import java.net.HttpURLConnection; +import java.net.URL; + +/** + * @author keli.wang + * @since 2017/9/1 + */ +public class MetaServerLocator { + private static final Logger LOG = LoggerFactory.getLogger(MetaServerLocator.class); + + private final String metaServerEndpoint; + + public MetaServerLocator(final String metaServerEndpoint) { + this.metaServerEndpoint = metaServerEndpoint; + } + + public Optional queryEndpoint() { + final String endpoint = request(); + if (endpoint == null || endpoint.length() == 0) { + LOG.error("meta server address list is empty!"); + return Optional.absent(); + } + + if (NetworkUtils.isValid(endpoint)) { + return Optional.of(endpoint); + } + return Optional.absent(); + } + + private String request() { + InputStreamReader in = null; + HttpURLConnection connection = null; + try { + connection = (HttpURLConnection) (new URL(metaServerEndpoint).openConnection()); + connection.setConnectTimeout(1000); + connection.setReadTimeout(500); + connection.setDoInput(true); + in = new InputStreamReader(connection.getInputStream()); + String content = CharStreams.toString(in); + if (connection.getResponseCode() != 200) { + return null; + } + return content.trim(); + } catch (IOException e) { + if (connection == null) { + return null; + } + InputStreamReader errIn = null; + try { + errIn = new InputStreamReader(connection.getErrorStream()); + String error = CharStreams.toString(errIn); + LOG.debug("read error stream {}", error); + } catch (IOException e1) { + LOG.debug("read error stream failed", e1); + } finally { + closeQuietly(errIn); + } + return null; + } catch (Exception e) { + return null; + } finally { + closeQuietly(in); + } + } + + private void closeQuietly(Closeable closeable) { + if (closeable == null) return; + try { + closeable.close(); + } catch (Exception e) { + LOG.debug("close failed"); + } + } +} diff --git a/qmq-common/src/main/java/qunar/tc/qmq/metrics/Metrics.java b/qmq-common/src/main/java/qunar/tc/qmq/metrics/Metrics.java new file mode 100644 index 00000000..c5a7d5d2 --- /dev/null +++ b/qmq-common/src/main/java/qunar/tc/qmq/metrics/Metrics.java @@ -0,0 +1,73 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.metrics; + +import com.google.common.base.Supplier; + +import java.util.ServiceLoader; + +public class Metrics { + private static final String[] EMPTY = new String[0]; + + private static final QmqMetricRegistry INSTANCE; + + static { + ServiceLoader registries = ServiceLoader.load(QmqMetricRegistry.class); + QmqMetricRegistry instance = null; + for (QmqMetricRegistry registry : registries) { + instance = registry; + break; + } + if (instance == null) { + instance = new MockRegistry(); + } + + INSTANCE = instance; + } + + public static void gauge(String name, String[] tags, String[] values, Supplier supplier) { + INSTANCE.newGauge(name, tags, values, supplier); + } + + public static void gauge(String name, Supplier supplier) { + INSTANCE.newGauge(name, EMPTY, EMPTY, supplier); + } + + public static QmqCounter counter(String name, String[] tags, String[] values) { + return INSTANCE.newCounter(name, tags, values); + } + + public static QmqCounter counter(String name) { + return INSTANCE.newCounter(name, EMPTY, EMPTY); + } + + public static QmqMeter meter(String name, String[] tags, String[] values) { + return INSTANCE.newMeter(name, tags, values); + } + + public static QmqTimer timer(String name, String[] tags, String[] values) { + return INSTANCE.newTimer(name, tags, values); + } + + public static QmqTimer timer(String name) { + return INSTANCE.newTimer(name, EMPTY, EMPTY); + } + + public static void remove(String name, String[] tags, String[] values) { + INSTANCE.remove(name, tags, values); + } +} \ No newline at end of file diff --git a/qmq-common/src/main/java/qunar/tc/qmq/metrics/MetricsConstants.java b/qmq-common/src/main/java/qunar/tc/qmq/metrics/MetricsConstants.java new file mode 100644 index 00000000..380c0aa8 --- /dev/null +++ b/qmq-common/src/main/java/qunar/tc/qmq/metrics/MetricsConstants.java @@ -0,0 +1,23 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.metrics; + +public class MetricsConstants { + public static final String[] SUBJECT_ARRAY = new String[]{"subject"}; + + public static final String[] SUBJECT_GROUP_ARRAY = new String[]{"subject", "group"}; +} diff --git a/qmq-common/src/main/java/qunar/tc/qmq/metrics/MockRegistry.java b/qmq-common/src/main/java/qunar/tc/qmq/metrics/MockRegistry.java new file mode 100644 index 00000000..17377922 --- /dev/null +++ b/qmq-common/src/main/java/qunar/tc/qmq/metrics/MockRegistry.java @@ -0,0 +1,99 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.metrics; + +import com.google.common.base.Supplier; + +import java.util.concurrent.TimeUnit; + +class MockRegistry implements QmqMetricRegistry { + + private static final QmqCounter COUNTER = new MockCounter(); + + private static final QmqMeter METER = new MockMeter(); + + private static final QmqTimer TIMER = new MockTimer(); + + @Override + public void newGauge(String name, String[] tags, String[] values, Supplier supplier) { + + } + + @Override + public QmqCounter newCounter(String name, String[] tags, String[] values) { + return COUNTER; + } + + @Override + public QmqMeter newMeter(String name, String[] tags, String[] values) { + return METER; + } + + @Override + public QmqTimer newTimer(String name, String[] tags, String[] values) { + return TIMER; + } + + @Override + public void remove(String name, String[] tags, String[] values) { + + } + + private static class MockCounter implements QmqCounter { + + @Override + public void inc() { + + } + + @Override + public void inc(long n) { + + } + + @Override + public void dec() { + + } + + @Override + public void dec(long n) { + + } + } + + private static class MockMeter implements QmqMeter { + + @Override + public void mark() { + + } + + @Override + public void mark(long n) { + + } + } + + private static class MockTimer implements QmqTimer { + + @Override + public void update(long duration, TimeUnit unit) { + + } + } +} diff --git a/qmq-common/src/main/java/qunar/tc/qmq/metrics/QmqCounter.java b/qmq-common/src/main/java/qunar/tc/qmq/metrics/QmqCounter.java new file mode 100644 index 00000000..dbd7cab4 --- /dev/null +++ b/qmq-common/src/main/java/qunar/tc/qmq/metrics/QmqCounter.java @@ -0,0 +1,31 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.metrics; + +/** + * @author keli.wang + * @since 2018/11/21 + */ +public interface QmqCounter { + void inc(); + + void inc(long n); + + void dec(); + + void dec(long n); +} diff --git a/qmq-common/src/main/java/qunar/tc/qmq/metrics/QmqMeter.java b/qmq-common/src/main/java/qunar/tc/qmq/metrics/QmqMeter.java new file mode 100644 index 00000000..a97fc1af --- /dev/null +++ b/qmq-common/src/main/java/qunar/tc/qmq/metrics/QmqMeter.java @@ -0,0 +1,27 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.metrics; + +/** + * @author keli.wang + * @since 2018/11/21 + */ +public interface QmqMeter { + void mark(); + + void mark(long n); +} diff --git a/qmq-common/src/main/java/qunar/tc/qmq/metrics/QmqMetricRegistry.java b/qmq-common/src/main/java/qunar/tc/qmq/metrics/QmqMetricRegistry.java new file mode 100644 index 00000000..cbafb450 --- /dev/null +++ b/qmq-common/src/main/java/qunar/tc/qmq/metrics/QmqMetricRegistry.java @@ -0,0 +1,35 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.metrics; + +import com.google.common.base.Supplier; + +/** + * @author keli.wang + * @since 2018/11/16 + */ +public interface QmqMetricRegistry { + void newGauge(final String name, final String[] tags, final String[] values, final Supplier supplier); + + QmqCounter newCounter(final String name, final String[] tags, final String[] values); + + QmqMeter newMeter(final String name, final String[] tags, final String[] values); + + QmqTimer newTimer(final String name, final String[] tags, final String[] values); + + void remove(final String name, final String[] tags, final String[] values); +} diff --git a/qmq-common/src/main/java/qunar/tc/qmq/metrics/QmqTimer.java b/qmq-common/src/main/java/qunar/tc/qmq/metrics/QmqTimer.java new file mode 100644 index 00000000..2a50a9bc --- /dev/null +++ b/qmq-common/src/main/java/qunar/tc/qmq/metrics/QmqTimer.java @@ -0,0 +1,27 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.metrics; + +import java.util.concurrent.TimeUnit; + +/** + * @author keli.wang + * @since 2018/11/21 + */ +public interface QmqTimer { + void update(long duration, TimeUnit unit); +} diff --git a/qmq-common/src/main/java/qunar/tc/qmq/service/exceptions/BlockMessageException.java b/qmq-common/src/main/java/qunar/tc/qmq/service/exceptions/BlockMessageException.java new file mode 100644 index 00000000..456d2865 --- /dev/null +++ b/qmq-common/src/main/java/qunar/tc/qmq/service/exceptions/BlockMessageException.java @@ -0,0 +1,30 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.service.exceptions; + +/** + * User: zhaohuiyu + * Date: 12/25/12 + * Time: 12:30 PM + */ +public class BlockMessageException extends MessageException { + private static final long serialVersionUID = 1068741830127606624L; + + public BlockMessageException(String messageId) { + super(messageId, "block message"); + } +} diff --git a/qmq-common/src/main/java/qunar/tc/qmq/service/exceptions/DuplicateMessageException.java b/qmq-common/src/main/java/qunar/tc/qmq/service/exceptions/DuplicateMessageException.java new file mode 100644 index 00000000..cee1dc5e --- /dev/null +++ b/qmq-common/src/main/java/qunar/tc/qmq/service/exceptions/DuplicateMessageException.java @@ -0,0 +1,31 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.service.exceptions; + +/** + * User: zhaohuiyu + * Date: 12/25/12 + * Time: 12:30 PM + */ +public class DuplicateMessageException extends MessageException { + + private static final long serialVersionUID = 8267606930373695631L; + + public DuplicateMessageException(String messageId) { + super(messageId, "Duplicated message"); + } +} diff --git a/qmq-common/src/main/java/qunar/tc/qmq/service/exceptions/MessageException.java b/qmq-common/src/main/java/qunar/tc/qmq/service/exceptions/MessageException.java new file mode 100644 index 00000000..c9e5f798 --- /dev/null +++ b/qmq-common/src/main/java/qunar/tc/qmq/service/exceptions/MessageException.java @@ -0,0 +1,66 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ +package qunar.tc.qmq.service.exceptions; + +/** + * @author miao.yang susing@gmail.com + * @date 2013-1-6 + */ +public class MessageException extends Exception { + + public static final String BROKER_BUSY = "broker busy"; + public static final String REJECT_MESSAGE = "message rejected"; + public static final String UNKONW_MESSAGE = "unkonwn exception"; + + private static final long serialVersionUID = -8385014158365588186L; + + private final String messageId; + + public MessageException(String messageId, String msg, Throwable t) { + super(msg, t); + this.messageId = messageId; + } + + public MessageException(String messageId, String msg) { + this(messageId, msg, null); + } + + public String getMessageId() { + return messageId; + } + + @Override + public Throwable initCause(Throwable cause) { + return this; + } + + @Override + public Throwable fillInStackTrace() { + return this; + } + + public boolean isBrokerBusy() { + return BROKER_BUSY.equals(getMessage()); + } + + public boolean isSubjectNotAssigned() { + return false; + } + + public boolean isRejected() { + return REJECT_MESSAGE.equals(getMessage()); + } +} diff --git a/qmq-common/src/main/java/qunar/tc/qmq/utils/CharsetUtils.java b/qmq-common/src/main/java/qunar/tc/qmq/utils/CharsetUtils.java new file mode 100644 index 00000000..1e6d5897 --- /dev/null +++ b/qmq-common/src/main/java/qunar/tc/qmq/utils/CharsetUtils.java @@ -0,0 +1,47 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.utils; + +import com.google.common.base.Strings; + +import java.io.UnsupportedEncodingException; +import java.nio.charset.Charset; + +/** + * @author yiqun.fan create on 17-7-6. + */ +public class CharsetUtils { + public static final Charset UTF8 = Charset.forName("utf-8"); + + private static final byte[] EMPTY_BYTES = new byte[0]; + + public static byte[] toUTF8Bytes(final String s) { + try { + return Strings.isNullOrEmpty(s) ? EMPTY_BYTES : s.getBytes("utf-8"); + } catch (UnsupportedEncodingException e) { + return null; + } + } + + public static String toUTF8String(final byte[] bs) { + try { + return bs == null || bs.length == 0 ? "" : new String(bs, "utf-8"); + } catch (UnsupportedEncodingException e) { + return ""; + } + } +} diff --git a/qmq-common/src/main/java/qunar/tc/qmq/utils/Checksums.java b/qmq-common/src/main/java/qunar/tc/qmq/utils/Checksums.java new file mode 100644 index 00000000..7f4f7ab5 --- /dev/null +++ b/qmq-common/src/main/java/qunar/tc/qmq/utils/Checksums.java @@ -0,0 +1,40 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.utils; + +import java.nio.ByteBuffer; +import java.util.zip.Checksum; + +final class Checksums { + + private Checksums() { + } + + public static void update(Checksum checksum, ByteBuffer buffer, int length) { + update(checksum, buffer, 0, length); + } + + public static void update(Checksum checksum, ByteBuffer buffer, int offset, int length) { + if (buffer.hasArray()) { + checksum.update(buffer.array(), buffer.position() + buffer.arrayOffset() + offset, length); + } else { + int start = buffer.position() + offset; + for (int i = start; i < start + length; i++) + checksum.update(buffer.get(i)); + } + } +} diff --git a/qmq-common/src/main/java/qunar/tc/qmq/utils/Crc32.java b/qmq-common/src/main/java/qunar/tc/qmq/utils/Crc32.java new file mode 100644 index 00000000..76d74997 --- /dev/null +++ b/qmq-common/src/main/java/qunar/tc/qmq/utils/Crc32.java @@ -0,0 +1,399 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.utils; + +import java.nio.ByteBuffer; +import java.util.zip.Checksum; + +/** + * This class was taken from Hadoop org.apache.hadoop.util.PureJavaCrc32 + * + * A pure-java implementation of the CRC32 checksum that uses the same polynomial as the built-in native CRC32. + * + * This is to avoid the JNI overhead for certain uses of Checksumming where many small pieces of data are checksummed in + * succession. + * + * The current version is ~10x to 1.8x as fast as Sun's native java.util.zip.CRC32 in Java 1.6 + * + * @see java.util.zip.CRC32 + */ +public class Crc32 implements Checksum { + + /** + * Compute the CRC32 of the byte array + * + * @param bytes The array to compute the checksum for + * @return The CRC32 + */ + public static long crc32(byte[] bytes) { + return crc32(bytes, 0, bytes.length); + } + + /** + * Compute the CRC32 of the segment of the byte array given by the specified size and offset + * + * @param bytes The bytes to checksum + * @param offset the offset at which to begin checksumming + * @param size the number of bytes to checksum + * @return The CRC32 + */ + public static long crc32(byte[] bytes, int offset, int size) { + Crc32 crc = new Crc32(); + crc.update(bytes, offset, size); + return crc.getValue(); + } + + /** + * Compute the CRC32 of a byte buffer from a given offset (relative to the buffer's current position) + * + * @param buffer The buffer with the underlying data + * @param offset The offset relative to the current position + * @param size The number of bytes beginning from the offset to include + * @return The CRC32 + */ + public static long crc32(ByteBuffer buffer, int offset, int size) { + Crc32 crc = new Crc32(); + Checksums.update(crc, buffer, offset, size); + return crc.getValue(); + } + + /** the current CRC value, bit-flipped */ + private int crc; + + /** Create a new PureJavaCrc32 object. */ + public Crc32() { + reset(); + } + + @Override + public long getValue() { + return (~crc) & 0xffffffffL; + } + + @Override + public void reset() { + crc = 0xffffffff; + } + + @Override + public void update(byte[] b, int off, int len) { + if (off < 0 || len < 0 || off > b.length - len) + throw new ArrayIndexOutOfBoundsException(); + + int localCrc = crc; + + while (len > 7) { + final int c0 = (b[off + 0] ^ localCrc) & 0xff; + final int c1 = (b[off + 1] ^ (localCrc >>>= 8)) & 0xff; + final int c2 = (b[off + 2] ^ (localCrc >>>= 8)) & 0xff; + final int c3 = (b[off + 3] ^ (localCrc >>>= 8)) & 0xff; + localCrc = (T[T8_7_START + c0] ^ T[T8_6_START + c1]) ^ (T[T8_5_START + c2] ^ T[T8_4_START + c3]); + + final int c4 = b[off + 4] & 0xff; + final int c5 = b[off + 5] & 0xff; + final int c6 = b[off + 6] & 0xff; + final int c7 = b[off + 7] & 0xff; + + localCrc ^= (T[T8_3_START + c4] ^ T[T8_2_START + c5]) ^ (T[T8_1_START + c6] ^ T[T8_0_START + c7]); + + off += 8; + len -= 8; + } + + /* loop unroll - duff's device style */ + switch (len) { + case 7: + localCrc = (localCrc >>> 8) ^ T[T8_0_START + ((localCrc ^ b[off++]) & 0xff)]; + case 6: + localCrc = (localCrc >>> 8) ^ T[T8_0_START + ((localCrc ^ b[off++]) & 0xff)]; + case 5: + localCrc = (localCrc >>> 8) ^ T[T8_0_START + ((localCrc ^ b[off++]) & 0xff)]; + case 4: + localCrc = (localCrc >>> 8) ^ T[T8_0_START + ((localCrc ^ b[off++]) & 0xff)]; + case 3: + localCrc = (localCrc >>> 8) ^ T[T8_0_START + ((localCrc ^ b[off++]) & 0xff)]; + case 2: + localCrc = (localCrc >>> 8) ^ T[T8_0_START + ((localCrc ^ b[off++]) & 0xff)]; + case 1: + localCrc = (localCrc >>> 8) ^ T[T8_0_START + ((localCrc ^ b[off++]) & 0xff)]; + default: + /* nothing */ + } + + // Publish crc out to object + crc = localCrc; + } + + @Override + final public void update(int b) { + crc = (crc >>> 8) ^ T[T8_0_START + ((crc ^ b) & 0xff)]; + } + + /* + * CRC-32 lookup tables generated by the polynomial 0xEDB88320. See also TestPureJavaCrc32.Table. + */ + private static final int T8_0_START = 0 * 256; + private static final int T8_1_START = 1 * 256; + private static final int T8_2_START = 2 * 256; + private static final int T8_3_START = 3 * 256; + private static final int T8_4_START = 4 * 256; + private static final int T8_5_START = 5 * 256; + private static final int T8_6_START = 6 * 256; + private static final int T8_7_START = 7 * 256; + + private static final int[] T = new int[] { + /* T8_0 */ + 0x00000000, 0x77073096, 0xEE0E612C, 0x990951BA, 0x076DC419, 0x706AF48F, 0xE963A535, 0x9E6495A3, 0x0EDB8832, + 0x79DCB8A4, 0xE0D5E91E, 0x97D2D988, 0x09B64C2B, 0x7EB17CBD, 0xE7B82D07, 0x90BF1D91, 0x1DB71064, 0x6AB020F2, + 0xF3B97148, 0x84BE41DE, 0x1ADAD47D, 0x6DDDE4EB, 0xF4D4B551, 0x83D385C7, 0x136C9856, 0x646BA8C0, 0xFD62F97A, + 0x8A65C9EC, 0x14015C4F, 0x63066CD9, 0xFA0F3D63, 0x8D080DF5, 0x3B6E20C8, 0x4C69105E, 0xD56041E4, 0xA2677172, + 0x3C03E4D1, 0x4B04D447, 0xD20D85FD, 0xA50AB56B, 0x35B5A8FA, 0x42B2986C, 0xDBBBC9D6, 0xACBCF940, 0x32D86CE3, + 0x45DF5C75, 0xDCD60DCF, 0xABD13D59, 0x26D930AC, 0x51DE003A, 0xC8D75180, 0xBFD06116, 0x21B4F4B5, 0x56B3C423, + 0xCFBA9599, 0xB8BDA50F, 0x2802B89E, 0x5F058808, 0xC60CD9B2, 0xB10BE924, 0x2F6F7C87, 0x58684C11, 0xC1611DAB, + 0xB6662D3D, 0x76DC4190, 0x01DB7106, 0x98D220BC, 0xEFD5102A, 0x71B18589, 0x06B6B51F, 0x9FBFE4A5, 0xE8B8D433, + 0x7807C9A2, 0x0F00F934, 0x9609A88E, 0xE10E9818, 0x7F6A0DBB, 0x086D3D2D, 0x91646C97, 0xE6635C01, 0x6B6B51F4, + 0x1C6C6162, 0x856530D8, 0xF262004E, 0x6C0695ED, 0x1B01A57B, 0x8208F4C1, 0xF50FC457, 0x65B0D9C6, 0x12B7E950, + 0x8BBEB8EA, 0xFCB9887C, 0x62DD1DDF, 0x15DA2D49, 0x8CD37CF3, 0xFBD44C65, 0x4DB26158, 0x3AB551CE, 0xA3BC0074, + 0xD4BB30E2, 0x4ADFA541, 0x3DD895D7, 0xA4D1C46D, 0xD3D6F4FB, 0x4369E96A, 0x346ED9FC, 0xAD678846, 0xDA60B8D0, + 0x44042D73, 0x33031DE5, 0xAA0A4C5F, 0xDD0D7CC9, 0x5005713C, 0x270241AA, 0xBE0B1010, 0xC90C2086, 0x5768B525, + 0x206F85B3, 0xB966D409, 0xCE61E49F, 0x5EDEF90E, 0x29D9C998, 0xB0D09822, 0xC7D7A8B4, 0x59B33D17, 0x2EB40D81, + 0xB7BD5C3B, 0xC0BA6CAD, 0xEDB88320, 0x9ABFB3B6, 0x03B6E20C, 0x74B1D29A, 0xEAD54739, 0x9DD277AF, 0x04DB2615, + 0x73DC1683, 0xE3630B12, 0x94643B84, 0x0D6D6A3E, 0x7A6A5AA8, 0xE40ECF0B, 0x9309FF9D, 0x0A00AE27, 0x7D079EB1, + 0xF00F9344, 0x8708A3D2, 0x1E01F268, 0x6906C2FE, 0xF762575D, 0x806567CB, 0x196C3671, 0x6E6B06E7, 0xFED41B76, + 0x89D32BE0, 0x10DA7A5A, 0x67DD4ACC, 0xF9B9DF6F, 0x8EBEEFF9, 0x17B7BE43, 0x60B08ED5, 0xD6D6A3E8, 0xA1D1937E, + 0x38D8C2C4, 0x4FDFF252, 0xD1BB67F1, 0xA6BC5767, 0x3FB506DD, 0x48B2364B, 0xD80D2BDA, 0xAF0A1B4C, 0x36034AF6, + 0x41047A60, 0xDF60EFC3, 0xA867DF55, 0x316E8EEF, 0x4669BE79, 0xCB61B38C, 0xBC66831A, 0x256FD2A0, 0x5268E236, + 0xCC0C7795, 0xBB0B4703, 0x220216B9, 0x5505262F, 0xC5BA3BBE, 0xB2BD0B28, 0x2BB45A92, 0x5CB36A04, 0xC2D7FFA7, + 0xB5D0CF31, 0x2CD99E8B, 0x5BDEAE1D, 0x9B64C2B0, 0xEC63F226, 0x756AA39C, 0x026D930A, 0x9C0906A9, 0xEB0E363F, + 0x72076785, 0x05005713, 0x95BF4A82, 0xE2B87A14, 0x7BB12BAE, 0x0CB61B38, 0x92D28E9B, 0xE5D5BE0D, 0x7CDCEFB7, + 0x0BDBDF21, 0x86D3D2D4, 0xF1D4E242, 0x68DDB3F8, 0x1FDA836E, 0x81BE16CD, 0xF6B9265B, 0x6FB077E1, 0x18B74777, + 0x88085AE6, 0xFF0F6A70, 0x66063BCA, 0x11010B5C, 0x8F659EFF, 0xF862AE69, 0x616BFFD3, 0x166CCF45, 0xA00AE278, + 0xD70DD2EE, 0x4E048354, 0x3903B3C2, 0xA7672661, 0xD06016F7, 0x4969474D, 0x3E6E77DB, 0xAED16A4A, 0xD9D65ADC, + 0x40DF0B66, 0x37D83BF0, 0xA9BCAE53, 0xDEBB9EC5, 0x47B2CF7F, 0x30B5FFE9, 0xBDBDF21C, 0xCABAC28A, 0x53B39330, + 0x24B4A3A6, 0xBAD03605, 0xCDD70693, 0x54DE5729, 0x23D967BF, 0xB3667A2E, 0xC4614AB8, 0x5D681B02, 0x2A6F2B94, + 0xB40BBE37, 0xC30C8EA1, 0x5A05DF1B, 0x2D02EF8D, + /* T8_1 */ + 0x00000000, 0x191B3141, 0x32366282, 0x2B2D53C3, 0x646CC504, 0x7D77F445, 0x565AA786, 0x4F4196C7, 0xC8D98A08, + 0xD1C2BB49, 0xFAEFE88A, 0xE3F4D9CB, 0xACB54F0C, 0xB5AE7E4D, 0x9E832D8E, 0x87981CCF, 0x4AC21251, 0x53D92310, + 0x78F470D3, 0x61EF4192, 0x2EAED755, 0x37B5E614, 0x1C98B5D7, 0x05838496, 0x821B9859, 0x9B00A918, 0xB02DFADB, + 0xA936CB9A, 0xE6775D5D, 0xFF6C6C1C, 0xD4413FDF, 0xCD5A0E9E, 0x958424A2, 0x8C9F15E3, 0xA7B24620, 0xBEA97761, + 0xF1E8E1A6, 0xE8F3D0E7, 0xC3DE8324, 0xDAC5B265, 0x5D5DAEAA, 0x44469FEB, 0x6F6BCC28, 0x7670FD69, 0x39316BAE, + 0x202A5AEF, 0x0B07092C, 0x121C386D, 0xDF4636F3, 0xC65D07B2, 0xED705471, 0xF46B6530, 0xBB2AF3F7, 0xA231C2B6, + 0x891C9175, 0x9007A034, 0x179FBCFB, 0x0E848DBA, 0x25A9DE79, 0x3CB2EF38, 0x73F379FF, 0x6AE848BE, 0x41C51B7D, + 0x58DE2A3C, 0xF0794F05, 0xE9627E44, 0xC24F2D87, 0xDB541CC6, 0x94158A01, 0x8D0EBB40, 0xA623E883, 0xBF38D9C2, + 0x38A0C50D, 0x21BBF44C, 0x0A96A78F, 0x138D96CE, 0x5CCC0009, 0x45D73148, 0x6EFA628B, 0x77E153CA, 0xBABB5D54, + 0xA3A06C15, 0x888D3FD6, 0x91960E97, 0xDED79850, 0xC7CCA911, 0xECE1FAD2, 0xF5FACB93, 0x7262D75C, 0x6B79E61D, + 0x4054B5DE, 0x594F849F, 0x160E1258, 0x0F152319, 0x243870DA, 0x3D23419B, 0x65FD6BA7, 0x7CE65AE6, 0x57CB0925, + 0x4ED03864, 0x0191AEA3, 0x188A9FE2, 0x33A7CC21, 0x2ABCFD60, 0xAD24E1AF, 0xB43FD0EE, 0x9F12832D, 0x8609B26C, + 0xC94824AB, 0xD05315EA, 0xFB7E4629, 0xE2657768, 0x2F3F79F6, 0x362448B7, 0x1D091B74, 0x04122A35, 0x4B53BCF2, + 0x52488DB3, 0x7965DE70, 0x607EEF31, 0xE7E6F3FE, 0xFEFDC2BF, 0xD5D0917C, 0xCCCBA03D, 0x838A36FA, 0x9A9107BB, + 0xB1BC5478, 0xA8A76539, 0x3B83984B, 0x2298A90A, 0x09B5FAC9, 0x10AECB88, 0x5FEF5D4F, 0x46F46C0E, 0x6DD93FCD, + 0x74C20E8C, 0xF35A1243, 0xEA412302, 0xC16C70C1, 0xD8774180, 0x9736D747, 0x8E2DE606, 0xA500B5C5, 0xBC1B8484, + 0x71418A1A, 0x685ABB5B, 0x4377E898, 0x5A6CD9D9, 0x152D4F1E, 0x0C367E5F, 0x271B2D9C, 0x3E001CDD, 0xB9980012, + 0xA0833153, 0x8BAE6290, 0x92B553D1, 0xDDF4C516, 0xC4EFF457, 0xEFC2A794, 0xF6D996D5, 0xAE07BCE9, 0xB71C8DA8, + 0x9C31DE6B, 0x852AEF2A, 0xCA6B79ED, 0xD37048AC, 0xF85D1B6F, 0xE1462A2E, 0x66DE36E1, 0x7FC507A0, 0x54E85463, + 0x4DF36522, 0x02B2F3E5, 0x1BA9C2A4, 0x30849167, 0x299FA026, 0xE4C5AEB8, 0xFDDE9FF9, 0xD6F3CC3A, 0xCFE8FD7B, + 0x80A96BBC, 0x99B25AFD, 0xB29F093E, 0xAB84387F, 0x2C1C24B0, 0x350715F1, 0x1E2A4632, 0x07317773, 0x4870E1B4, + 0x516BD0F5, 0x7A468336, 0x635DB277, 0xCBFAD74E, 0xD2E1E60F, 0xF9CCB5CC, 0xE0D7848D, 0xAF96124A, 0xB68D230B, + 0x9DA070C8, 0x84BB4189, 0x03235D46, 0x1A386C07, 0x31153FC4, 0x280E0E85, 0x674F9842, 0x7E54A903, 0x5579FAC0, + 0x4C62CB81, 0x8138C51F, 0x9823F45E, 0xB30EA79D, 0xAA1596DC, 0xE554001B, 0xFC4F315A, 0xD7626299, 0xCE7953D8, + 0x49E14F17, 0x50FA7E56, 0x7BD72D95, 0x62CC1CD4, 0x2D8D8A13, 0x3496BB52, 0x1FBBE891, 0x06A0D9D0, 0x5E7EF3EC, + 0x4765C2AD, 0x6C48916E, 0x7553A02F, 0x3A1236E8, 0x230907A9, 0x0824546A, 0x113F652B, 0x96A779E4, 0x8FBC48A5, + 0xA4911B66, 0xBD8A2A27, 0xF2CBBCE0, 0xEBD08DA1, 0xC0FDDE62, 0xD9E6EF23, 0x14BCE1BD, 0x0DA7D0FC, 0x268A833F, + 0x3F91B27E, 0x70D024B9, 0x69CB15F8, 0x42E6463B, 0x5BFD777A, 0xDC656BB5, 0xC57E5AF4, 0xEE530937, 0xF7483876, + 0xB809AEB1, 0xA1129FF0, 0x8A3FCC33, 0x9324FD72, + /* T8_2 */ + 0x00000000, 0x01C26A37, 0x0384D46E, 0x0246BE59, 0x0709A8DC, 0x06CBC2EB, 0x048D7CB2, 0x054F1685, 0x0E1351B8, + 0x0FD13B8F, 0x0D9785D6, 0x0C55EFE1, 0x091AF964, 0x08D89353, 0x0A9E2D0A, 0x0B5C473D, 0x1C26A370, 0x1DE4C947, + 0x1FA2771E, 0x1E601D29, 0x1B2F0BAC, 0x1AED619B, 0x18ABDFC2, 0x1969B5F5, 0x1235F2C8, 0x13F798FF, 0x11B126A6, + 0x10734C91, 0x153C5A14, 0x14FE3023, 0x16B88E7A, 0x177AE44D, 0x384D46E0, 0x398F2CD7, 0x3BC9928E, 0x3A0BF8B9, + 0x3F44EE3C, 0x3E86840B, 0x3CC03A52, 0x3D025065, 0x365E1758, 0x379C7D6F, 0x35DAC336, 0x3418A901, 0x3157BF84, + 0x3095D5B3, 0x32D36BEA, 0x331101DD, 0x246BE590, 0x25A98FA7, 0x27EF31FE, 0x262D5BC9, 0x23624D4C, 0x22A0277B, + 0x20E69922, 0x2124F315, 0x2A78B428, 0x2BBADE1F, 0x29FC6046, 0x283E0A71, 0x2D711CF4, 0x2CB376C3, 0x2EF5C89A, + 0x2F37A2AD, 0x709A8DC0, 0x7158E7F7, 0x731E59AE, 0x72DC3399, 0x7793251C, 0x76514F2B, 0x7417F172, 0x75D59B45, + 0x7E89DC78, 0x7F4BB64F, 0x7D0D0816, 0x7CCF6221, 0x798074A4, 0x78421E93, 0x7A04A0CA, 0x7BC6CAFD, 0x6CBC2EB0, + 0x6D7E4487, 0x6F38FADE, 0x6EFA90E9, 0x6BB5866C, 0x6A77EC5B, 0x68315202, 0x69F33835, 0x62AF7F08, 0x636D153F, + 0x612BAB66, 0x60E9C151, 0x65A6D7D4, 0x6464BDE3, 0x662203BA, 0x67E0698D, 0x48D7CB20, 0x4915A117, 0x4B531F4E, + 0x4A917579, 0x4FDE63FC, 0x4E1C09CB, 0x4C5AB792, 0x4D98DDA5, 0x46C49A98, 0x4706F0AF, 0x45404EF6, 0x448224C1, + 0x41CD3244, 0x400F5873, 0x4249E62A, 0x438B8C1D, 0x54F16850, 0x55330267, 0x5775BC3E, 0x56B7D609, 0x53F8C08C, + 0x523AAABB, 0x507C14E2, 0x51BE7ED5, 0x5AE239E8, 0x5B2053DF, 0x5966ED86, 0x58A487B1, 0x5DEB9134, 0x5C29FB03, + 0x5E6F455A, 0x5FAD2F6D, 0xE1351B80, 0xE0F771B7, 0xE2B1CFEE, 0xE373A5D9, 0xE63CB35C, 0xE7FED96B, 0xE5B86732, + 0xE47A0D05, 0xEF264A38, 0xEEE4200F, 0xECA29E56, 0xED60F461, 0xE82FE2E4, 0xE9ED88D3, 0xEBAB368A, 0xEA695CBD, + 0xFD13B8F0, 0xFCD1D2C7, 0xFE976C9E, 0xFF5506A9, 0xFA1A102C, 0xFBD87A1B, 0xF99EC442, 0xF85CAE75, 0xF300E948, + 0xF2C2837F, 0xF0843D26, 0xF1465711, 0xF4094194, 0xF5CB2BA3, 0xF78D95FA, 0xF64FFFCD, 0xD9785D60, 0xD8BA3757, + 0xDAFC890E, 0xDB3EE339, 0xDE71F5BC, 0xDFB39F8B, 0xDDF521D2, 0xDC374BE5, 0xD76B0CD8, 0xD6A966EF, 0xD4EFD8B6, + 0xD52DB281, 0xD062A404, 0xD1A0CE33, 0xD3E6706A, 0xD2241A5D, 0xC55EFE10, 0xC49C9427, 0xC6DA2A7E, 0xC7184049, + 0xC25756CC, 0xC3953CFB, 0xC1D382A2, 0xC011E895, 0xCB4DAFA8, 0xCA8FC59F, 0xC8C97BC6, 0xC90B11F1, 0xCC440774, + 0xCD866D43, 0xCFC0D31A, 0xCE02B92D, 0x91AF9640, 0x906DFC77, 0x922B422E, 0x93E92819, 0x96A63E9C, 0x976454AB, + 0x9522EAF2, 0x94E080C5, 0x9FBCC7F8, 0x9E7EADCF, 0x9C381396, 0x9DFA79A1, 0x98B56F24, 0x99770513, 0x9B31BB4A, + 0x9AF3D17D, 0x8D893530, 0x8C4B5F07, 0x8E0DE15E, 0x8FCF8B69, 0x8A809DEC, 0x8B42F7DB, 0x89044982, 0x88C623B5, + 0x839A6488, 0x82580EBF, 0x801EB0E6, 0x81DCDAD1, 0x8493CC54, 0x8551A663, 0x8717183A, 0x86D5720D, 0xA9E2D0A0, + 0xA820BA97, 0xAA6604CE, 0xABA46EF9, 0xAEEB787C, 0xAF29124B, 0xAD6FAC12, 0xACADC625, 0xA7F18118, 0xA633EB2F, + 0xA4755576, 0xA5B73F41, 0xA0F829C4, 0xA13A43F3, 0xA37CFDAA, 0xA2BE979D, 0xB5C473D0, 0xB40619E7, 0xB640A7BE, + 0xB782CD89, 0xB2CDDB0C, 0xB30FB13B, 0xB1490F62, 0xB08B6555, 0xBBD72268, 0xBA15485F, 0xB853F606, 0xB9919C31, + 0xBCDE8AB4, 0xBD1CE083, 0xBF5A5EDA, 0xBE9834ED, + /* T8_3 */ + 0x00000000, 0xB8BC6765, 0xAA09C88B, 0x12B5AFEE, 0x8F629757, 0x37DEF032, 0x256B5FDC, 0x9DD738B9, 0xC5B428EF, + 0x7D084F8A, 0x6FBDE064, 0xD7018701, 0x4AD6BFB8, 0xF26AD8DD, 0xE0DF7733, 0x58631056, 0x5019579F, 0xE8A530FA, + 0xFA109F14, 0x42ACF871, 0xDF7BC0C8, 0x67C7A7AD, 0x75720843, 0xCDCE6F26, 0x95AD7F70, 0x2D111815, 0x3FA4B7FB, + 0x8718D09E, 0x1ACFE827, 0xA2738F42, 0xB0C620AC, 0x087A47C9, 0xA032AF3E, 0x188EC85B, 0x0A3B67B5, 0xB28700D0, + 0x2F503869, 0x97EC5F0C, 0x8559F0E2, 0x3DE59787, 0x658687D1, 0xDD3AE0B4, 0xCF8F4F5A, 0x7733283F, 0xEAE41086, + 0x525877E3, 0x40EDD80D, 0xF851BF68, 0xF02BF8A1, 0x48979FC4, 0x5A22302A, 0xE29E574F, 0x7F496FF6, 0xC7F50893, + 0xD540A77D, 0x6DFCC018, 0x359FD04E, 0x8D23B72B, 0x9F9618C5, 0x272A7FA0, 0xBAFD4719, 0x0241207C, 0x10F48F92, + 0xA848E8F7, 0x9B14583D, 0x23A83F58, 0x311D90B6, 0x89A1F7D3, 0x1476CF6A, 0xACCAA80F, 0xBE7F07E1, 0x06C36084, + 0x5EA070D2, 0xE61C17B7, 0xF4A9B859, 0x4C15DF3C, 0xD1C2E785, 0x697E80E0, 0x7BCB2F0E, 0xC377486B, 0xCB0D0FA2, + 0x73B168C7, 0x6104C729, 0xD9B8A04C, 0x446F98F5, 0xFCD3FF90, 0xEE66507E, 0x56DA371B, 0x0EB9274D, 0xB6054028, + 0xA4B0EFC6, 0x1C0C88A3, 0x81DBB01A, 0x3967D77F, 0x2BD27891, 0x936E1FF4, 0x3B26F703, 0x839A9066, 0x912F3F88, + 0x299358ED, 0xB4446054, 0x0CF80731, 0x1E4DA8DF, 0xA6F1CFBA, 0xFE92DFEC, 0x462EB889, 0x549B1767, 0xEC277002, + 0x71F048BB, 0xC94C2FDE, 0xDBF98030, 0x6345E755, 0x6B3FA09C, 0xD383C7F9, 0xC1366817, 0x798A0F72, 0xE45D37CB, + 0x5CE150AE, 0x4E54FF40, 0xF6E89825, 0xAE8B8873, 0x1637EF16, 0x048240F8, 0xBC3E279D, 0x21E91F24, 0x99557841, + 0x8BE0D7AF, 0x335CB0CA, 0xED59B63B, 0x55E5D15E, 0x47507EB0, 0xFFEC19D5, 0x623B216C, 0xDA874609, 0xC832E9E7, + 0x708E8E82, 0x28ED9ED4, 0x9051F9B1, 0x82E4565F, 0x3A58313A, 0xA78F0983, 0x1F336EE6, 0x0D86C108, 0xB53AA66D, + 0xBD40E1A4, 0x05FC86C1, 0x1749292F, 0xAFF54E4A, 0x322276F3, 0x8A9E1196, 0x982BBE78, 0x2097D91D, 0x78F4C94B, + 0xC048AE2E, 0xD2FD01C0, 0x6A4166A5, 0xF7965E1C, 0x4F2A3979, 0x5D9F9697, 0xE523F1F2, 0x4D6B1905, 0xF5D77E60, + 0xE762D18E, 0x5FDEB6EB, 0xC2098E52, 0x7AB5E937, 0x680046D9, 0xD0BC21BC, 0x88DF31EA, 0x3063568F, 0x22D6F961, + 0x9A6A9E04, 0x07BDA6BD, 0xBF01C1D8, 0xADB46E36, 0x15080953, 0x1D724E9A, 0xA5CE29FF, 0xB77B8611, 0x0FC7E174, + 0x9210D9CD, 0x2AACBEA8, 0x38191146, 0x80A57623, 0xD8C66675, 0x607A0110, 0x72CFAEFE, 0xCA73C99B, 0x57A4F122, + 0xEF189647, 0xFDAD39A9, 0x45115ECC, 0x764DEE06, 0xCEF18963, 0xDC44268D, 0x64F841E8, 0xF92F7951, 0x41931E34, + 0x5326B1DA, 0xEB9AD6BF, 0xB3F9C6E9, 0x0B45A18C, 0x19F00E62, 0xA14C6907, 0x3C9B51BE, 0x842736DB, 0x96929935, + 0x2E2EFE50, 0x2654B999, 0x9EE8DEFC, 0x8C5D7112, 0x34E11677, 0xA9362ECE, 0x118A49AB, 0x033FE645, 0xBB838120, + 0xE3E09176, 0x5B5CF613, 0x49E959FD, 0xF1553E98, 0x6C820621, 0xD43E6144, 0xC68BCEAA, 0x7E37A9CF, 0xD67F4138, + 0x6EC3265D, 0x7C7689B3, 0xC4CAEED6, 0x591DD66F, 0xE1A1B10A, 0xF3141EE4, 0x4BA87981, 0x13CB69D7, 0xAB770EB2, + 0xB9C2A15C, 0x017EC639, 0x9CA9FE80, 0x241599E5, 0x36A0360B, 0x8E1C516E, 0x866616A7, 0x3EDA71C2, 0x2C6FDE2C, + 0x94D3B949, 0x090481F0, 0xB1B8E695, 0xA30D497B, 0x1BB12E1E, 0x43D23E48, 0xFB6E592D, 0xE9DBF6C3, 0x516791A6, + 0xCCB0A91F, 0x740CCE7A, 0x66B96194, 0xDE0506F1, + /* T8_4 */ + 0x00000000, 0x3D6029B0, 0x7AC05360, 0x47A07AD0, 0xF580A6C0, 0xC8E08F70, 0x8F40F5A0, 0xB220DC10, 0x30704BC1, + 0x0D106271, 0x4AB018A1, 0x77D03111, 0xC5F0ED01, 0xF890C4B1, 0xBF30BE61, 0x825097D1, 0x60E09782, 0x5D80BE32, + 0x1A20C4E2, 0x2740ED52, 0x95603142, 0xA80018F2, 0xEFA06222, 0xD2C04B92, 0x5090DC43, 0x6DF0F5F3, 0x2A508F23, + 0x1730A693, 0xA5107A83, 0x98705333, 0xDFD029E3, 0xE2B00053, 0xC1C12F04, 0xFCA106B4, 0xBB017C64, 0x866155D4, + 0x344189C4, 0x0921A074, 0x4E81DAA4, 0x73E1F314, 0xF1B164C5, 0xCCD14D75, 0x8B7137A5, 0xB6111E15, 0x0431C205, + 0x3951EBB5, 0x7EF19165, 0x4391B8D5, 0xA121B886, 0x9C419136, 0xDBE1EBE6, 0xE681C256, 0x54A11E46, 0x69C137F6, + 0x2E614D26, 0x13016496, 0x9151F347, 0xAC31DAF7, 0xEB91A027, 0xD6F18997, 0x64D15587, 0x59B17C37, 0x1E1106E7, + 0x23712F57, 0x58F35849, 0x659371F9, 0x22330B29, 0x1F532299, 0xAD73FE89, 0x9013D739, 0xD7B3ADE9, 0xEAD38459, + 0x68831388, 0x55E33A38, 0x124340E8, 0x2F236958, 0x9D03B548, 0xA0639CF8, 0xE7C3E628, 0xDAA3CF98, 0x3813CFCB, + 0x0573E67B, 0x42D39CAB, 0x7FB3B51B, 0xCD93690B, 0xF0F340BB, 0xB7533A6B, 0x8A3313DB, 0x0863840A, 0x3503ADBA, + 0x72A3D76A, 0x4FC3FEDA, 0xFDE322CA, 0xC0830B7A, 0x872371AA, 0xBA43581A, 0x9932774D, 0xA4525EFD, 0xE3F2242D, + 0xDE920D9D, 0x6CB2D18D, 0x51D2F83D, 0x167282ED, 0x2B12AB5D, 0xA9423C8C, 0x9422153C, 0xD3826FEC, 0xEEE2465C, + 0x5CC29A4C, 0x61A2B3FC, 0x2602C92C, 0x1B62E09C, 0xF9D2E0CF, 0xC4B2C97F, 0x8312B3AF, 0xBE729A1F, 0x0C52460F, + 0x31326FBF, 0x7692156F, 0x4BF23CDF, 0xC9A2AB0E, 0xF4C282BE, 0xB362F86E, 0x8E02D1DE, 0x3C220DCE, 0x0142247E, + 0x46E25EAE, 0x7B82771E, 0xB1E6B092, 0x8C869922, 0xCB26E3F2, 0xF646CA42, 0x44661652, 0x79063FE2, 0x3EA64532, + 0x03C66C82, 0x8196FB53, 0xBCF6D2E3, 0xFB56A833, 0xC6368183, 0x74165D93, 0x49767423, 0x0ED60EF3, 0x33B62743, + 0xD1062710, 0xEC660EA0, 0xABC67470, 0x96A65DC0, 0x248681D0, 0x19E6A860, 0x5E46D2B0, 0x6326FB00, 0xE1766CD1, + 0xDC164561, 0x9BB63FB1, 0xA6D61601, 0x14F6CA11, 0x2996E3A1, 0x6E369971, 0x5356B0C1, 0x70279F96, 0x4D47B626, + 0x0AE7CCF6, 0x3787E546, 0x85A73956, 0xB8C710E6, 0xFF676A36, 0xC2074386, 0x4057D457, 0x7D37FDE7, 0x3A978737, + 0x07F7AE87, 0xB5D77297, 0x88B75B27, 0xCF1721F7, 0xF2770847, 0x10C70814, 0x2DA721A4, 0x6A075B74, 0x576772C4, + 0xE547AED4, 0xD8278764, 0x9F87FDB4, 0xA2E7D404, 0x20B743D5, 0x1DD76A65, 0x5A7710B5, 0x67173905, 0xD537E515, + 0xE857CCA5, 0xAFF7B675, 0x92979FC5, 0xE915E8DB, 0xD475C16B, 0x93D5BBBB, 0xAEB5920B, 0x1C954E1B, 0x21F567AB, + 0x66551D7B, 0x5B3534CB, 0xD965A31A, 0xE4058AAA, 0xA3A5F07A, 0x9EC5D9CA, 0x2CE505DA, 0x11852C6A, 0x562556BA, + 0x6B457F0A, 0x89F57F59, 0xB49556E9, 0xF3352C39, 0xCE550589, 0x7C75D999, 0x4115F029, 0x06B58AF9, 0x3BD5A349, + 0xB9853498, 0x84E51D28, 0xC34567F8, 0xFE254E48, 0x4C059258, 0x7165BBE8, 0x36C5C138, 0x0BA5E888, 0x28D4C7DF, + 0x15B4EE6F, 0x521494BF, 0x6F74BD0F, 0xDD54611F, 0xE03448AF, 0xA794327F, 0x9AF41BCF, 0x18A48C1E, 0x25C4A5AE, + 0x6264DF7E, 0x5F04F6CE, 0xED242ADE, 0xD044036E, 0x97E479BE, 0xAA84500E, 0x4834505D, 0x755479ED, 0x32F4033D, + 0x0F942A8D, 0xBDB4F69D, 0x80D4DF2D, 0xC774A5FD, 0xFA148C4D, 0x78441B9C, 0x4524322C, 0x028448FC, 0x3FE4614C, + 0x8DC4BD5C, 0xB0A494EC, 0xF704EE3C, 0xCA64C78C, + /* T8_5 */ + 0x00000000, 0xCB5CD3A5, 0x4DC8A10B, 0x869472AE, 0x9B914216, 0x50CD91B3, 0xD659E31D, 0x1D0530B8, 0xEC53826D, + 0x270F51C8, 0xA19B2366, 0x6AC7F0C3, 0x77C2C07B, 0xBC9E13DE, 0x3A0A6170, 0xF156B2D5, 0x03D6029B, 0xC88AD13E, + 0x4E1EA390, 0x85427035, 0x9847408D, 0x531B9328, 0xD58FE186, 0x1ED33223, 0xEF8580F6, 0x24D95353, 0xA24D21FD, + 0x6911F258, 0x7414C2E0, 0xBF481145, 0x39DC63EB, 0xF280B04E, 0x07AC0536, 0xCCF0D693, 0x4A64A43D, 0x81387798, + 0x9C3D4720, 0x57619485, 0xD1F5E62B, 0x1AA9358E, 0xEBFF875B, 0x20A354FE, 0xA6372650, 0x6D6BF5F5, 0x706EC54D, + 0xBB3216E8, 0x3DA66446, 0xF6FAB7E3, 0x047A07AD, 0xCF26D408, 0x49B2A6A6, 0x82EE7503, 0x9FEB45BB, 0x54B7961E, + 0xD223E4B0, 0x197F3715, 0xE82985C0, 0x23755665, 0xA5E124CB, 0x6EBDF76E, 0x73B8C7D6, 0xB8E41473, 0x3E7066DD, + 0xF52CB578, 0x0F580A6C, 0xC404D9C9, 0x4290AB67, 0x89CC78C2, 0x94C9487A, 0x5F959BDF, 0xD901E971, 0x125D3AD4, + 0xE30B8801, 0x28575BA4, 0xAEC3290A, 0x659FFAAF, 0x789ACA17, 0xB3C619B2, 0x35526B1C, 0xFE0EB8B9, 0x0C8E08F7, + 0xC7D2DB52, 0x4146A9FC, 0x8A1A7A59, 0x971F4AE1, 0x5C439944, 0xDAD7EBEA, 0x118B384F, 0xE0DD8A9A, 0x2B81593F, + 0xAD152B91, 0x6649F834, 0x7B4CC88C, 0xB0101B29, 0x36846987, 0xFDD8BA22, 0x08F40F5A, 0xC3A8DCFF, 0x453CAE51, + 0x8E607DF4, 0x93654D4C, 0x58399EE9, 0xDEADEC47, 0x15F13FE2, 0xE4A78D37, 0x2FFB5E92, 0xA96F2C3C, 0x6233FF99, + 0x7F36CF21, 0xB46A1C84, 0x32FE6E2A, 0xF9A2BD8F, 0x0B220DC1, 0xC07EDE64, 0x46EAACCA, 0x8DB67F6F, 0x90B34FD7, + 0x5BEF9C72, 0xDD7BEEDC, 0x16273D79, 0xE7718FAC, 0x2C2D5C09, 0xAAB92EA7, 0x61E5FD02, 0x7CE0CDBA, 0xB7BC1E1F, + 0x31286CB1, 0xFA74BF14, 0x1EB014D8, 0xD5ECC77D, 0x5378B5D3, 0x98246676, 0x852156CE, 0x4E7D856B, 0xC8E9F7C5, + 0x03B52460, 0xF2E396B5, 0x39BF4510, 0xBF2B37BE, 0x7477E41B, 0x6972D4A3, 0xA22E0706, 0x24BA75A8, 0xEFE6A60D, + 0x1D661643, 0xD63AC5E6, 0x50AEB748, 0x9BF264ED, 0x86F75455, 0x4DAB87F0, 0xCB3FF55E, 0x006326FB, 0xF135942E, + 0x3A69478B, 0xBCFD3525, 0x77A1E680, 0x6AA4D638, 0xA1F8059D, 0x276C7733, 0xEC30A496, 0x191C11EE, 0xD240C24B, + 0x54D4B0E5, 0x9F886340, 0x828D53F8, 0x49D1805D, 0xCF45F2F3, 0x04192156, 0xF54F9383, 0x3E134026, 0xB8873288, + 0x73DBE12D, 0x6EDED195, 0xA5820230, 0x2316709E, 0xE84AA33B, 0x1ACA1375, 0xD196C0D0, 0x5702B27E, 0x9C5E61DB, + 0x815B5163, 0x4A0782C6, 0xCC93F068, 0x07CF23CD, 0xF6999118, 0x3DC542BD, 0xBB513013, 0x700DE3B6, 0x6D08D30E, + 0xA65400AB, 0x20C07205, 0xEB9CA1A0, 0x11E81EB4, 0xDAB4CD11, 0x5C20BFBF, 0x977C6C1A, 0x8A795CA2, 0x41258F07, + 0xC7B1FDA9, 0x0CED2E0C, 0xFDBB9CD9, 0x36E74F7C, 0xB0733DD2, 0x7B2FEE77, 0x662ADECF, 0xAD760D6A, 0x2BE27FC4, + 0xE0BEAC61, 0x123E1C2F, 0xD962CF8A, 0x5FF6BD24, 0x94AA6E81, 0x89AF5E39, 0x42F38D9C, 0xC467FF32, 0x0F3B2C97, + 0xFE6D9E42, 0x35314DE7, 0xB3A53F49, 0x78F9ECEC, 0x65FCDC54, 0xAEA00FF1, 0x28347D5F, 0xE368AEFA, 0x16441B82, + 0xDD18C827, 0x5B8CBA89, 0x90D0692C, 0x8DD55994, 0x46898A31, 0xC01DF89F, 0x0B412B3A, 0xFA1799EF, 0x314B4A4A, + 0xB7DF38E4, 0x7C83EB41, 0x6186DBF9, 0xAADA085C, 0x2C4E7AF2, 0xE712A957, 0x15921919, 0xDECECABC, 0x585AB812, + 0x93066BB7, 0x8E035B0F, 0x455F88AA, 0xC3CBFA04, 0x089729A1, 0xF9C19B74, 0x329D48D1, 0xB4093A7F, 0x7F55E9DA, + 0x6250D962, 0xA90C0AC7, 0x2F987869, 0xE4C4ABCC, + /* T8_6 */ + 0x00000000, 0xA6770BB4, 0x979F1129, 0x31E81A9D, 0xF44F2413, 0x52382FA7, 0x63D0353A, 0xC5A73E8E, 0x33EF4E67, + 0x959845D3, 0xA4705F4E, 0x020754FA, 0xC7A06A74, 0x61D761C0, 0x503F7B5D, 0xF64870E9, 0x67DE9CCE, 0xC1A9977A, + 0xF0418DE7, 0x56368653, 0x9391B8DD, 0x35E6B369, 0x040EA9F4, 0xA279A240, 0x5431D2A9, 0xF246D91D, 0xC3AEC380, + 0x65D9C834, 0xA07EF6BA, 0x0609FD0E, 0x37E1E793, 0x9196EC27, 0xCFBD399C, 0x69CA3228, 0x582228B5, 0xFE552301, + 0x3BF21D8F, 0x9D85163B, 0xAC6D0CA6, 0x0A1A0712, 0xFC5277FB, 0x5A257C4F, 0x6BCD66D2, 0xCDBA6D66, 0x081D53E8, + 0xAE6A585C, 0x9F8242C1, 0x39F54975, 0xA863A552, 0x0E14AEE6, 0x3FFCB47B, 0x998BBFCF, 0x5C2C8141, 0xFA5B8AF5, + 0xCBB39068, 0x6DC49BDC, 0x9B8CEB35, 0x3DFBE081, 0x0C13FA1C, 0xAA64F1A8, 0x6FC3CF26, 0xC9B4C492, 0xF85CDE0F, + 0x5E2BD5BB, 0x440B7579, 0xE27C7ECD, 0xD3946450, 0x75E36FE4, 0xB044516A, 0x16335ADE, 0x27DB4043, 0x81AC4BF7, + 0x77E43B1E, 0xD19330AA, 0xE07B2A37, 0x460C2183, 0x83AB1F0D, 0x25DC14B9, 0x14340E24, 0xB2430590, 0x23D5E9B7, + 0x85A2E203, 0xB44AF89E, 0x123DF32A, 0xD79ACDA4, 0x71EDC610, 0x4005DC8D, 0xE672D739, 0x103AA7D0, 0xB64DAC64, + 0x87A5B6F9, 0x21D2BD4D, 0xE47583C3, 0x42028877, 0x73EA92EA, 0xD59D995E, 0x8BB64CE5, 0x2DC14751, 0x1C295DCC, + 0xBA5E5678, 0x7FF968F6, 0xD98E6342, 0xE86679DF, 0x4E11726B, 0xB8590282, 0x1E2E0936, 0x2FC613AB, 0x89B1181F, + 0x4C162691, 0xEA612D25, 0xDB8937B8, 0x7DFE3C0C, 0xEC68D02B, 0x4A1FDB9F, 0x7BF7C102, 0xDD80CAB6, 0x1827F438, + 0xBE50FF8C, 0x8FB8E511, 0x29CFEEA5, 0xDF879E4C, 0x79F095F8, 0x48188F65, 0xEE6F84D1, 0x2BC8BA5F, 0x8DBFB1EB, + 0xBC57AB76, 0x1A20A0C2, 0x8816EAF2, 0x2E61E146, 0x1F89FBDB, 0xB9FEF06F, 0x7C59CEE1, 0xDA2EC555, 0xEBC6DFC8, + 0x4DB1D47C, 0xBBF9A495, 0x1D8EAF21, 0x2C66B5BC, 0x8A11BE08, 0x4FB68086, 0xE9C18B32, 0xD82991AF, 0x7E5E9A1B, + 0xEFC8763C, 0x49BF7D88, 0x78576715, 0xDE206CA1, 0x1B87522F, 0xBDF0599B, 0x8C184306, 0x2A6F48B2, 0xDC27385B, + 0x7A5033EF, 0x4BB82972, 0xEDCF22C6, 0x28681C48, 0x8E1F17FC, 0xBFF70D61, 0x198006D5, 0x47ABD36E, 0xE1DCD8DA, + 0xD034C247, 0x7643C9F3, 0xB3E4F77D, 0x1593FCC9, 0x247BE654, 0x820CEDE0, 0x74449D09, 0xD23396BD, 0xE3DB8C20, + 0x45AC8794, 0x800BB91A, 0x267CB2AE, 0x1794A833, 0xB1E3A387, 0x20754FA0, 0x86024414, 0xB7EA5E89, 0x119D553D, + 0xD43A6BB3, 0x724D6007, 0x43A57A9A, 0xE5D2712E, 0x139A01C7, 0xB5ED0A73, 0x840510EE, 0x22721B5A, 0xE7D525D4, + 0x41A22E60, 0x704A34FD, 0xD63D3F49, 0xCC1D9F8B, 0x6A6A943F, 0x5B828EA2, 0xFDF58516, 0x3852BB98, 0x9E25B02C, + 0xAFCDAAB1, 0x09BAA105, 0xFFF2D1EC, 0x5985DA58, 0x686DC0C5, 0xCE1ACB71, 0x0BBDF5FF, 0xADCAFE4B, 0x9C22E4D6, + 0x3A55EF62, 0xABC30345, 0x0DB408F1, 0x3C5C126C, 0x9A2B19D8, 0x5F8C2756, 0xF9FB2CE2, 0xC813367F, 0x6E643DCB, + 0x982C4D22, 0x3E5B4696, 0x0FB35C0B, 0xA9C457BF, 0x6C636931, 0xCA146285, 0xFBFC7818, 0x5D8B73AC, 0x03A0A617, + 0xA5D7ADA3, 0x943FB73E, 0x3248BC8A, 0xF7EF8204, 0x519889B0, 0x6070932D, 0xC6079899, 0x304FE870, 0x9638E3C4, + 0xA7D0F959, 0x01A7F2ED, 0xC400CC63, 0x6277C7D7, 0x539FDD4A, 0xF5E8D6FE, 0x647E3AD9, 0xC209316D, 0xF3E12BF0, + 0x55962044, 0x90311ECA, 0x3646157E, 0x07AE0FE3, 0xA1D90457, 0x579174BE, 0xF1E67F0A, 0xC00E6597, 0x66796E23, + 0xA3DE50AD, 0x05A95B19, 0x34414184, 0x92364A30, + /* T8_7 */ + 0x00000000, 0xCCAA009E, 0x4225077D, 0x8E8F07E3, 0x844A0EFA, 0x48E00E64, 0xC66F0987, 0x0AC50919, 0xD3E51BB5, + 0x1F4F1B2B, 0x91C01CC8, 0x5D6A1C56, 0x57AF154F, 0x9B0515D1, 0x158A1232, 0xD92012AC, 0x7CBB312B, 0xB01131B5, + 0x3E9E3656, 0xF23436C8, 0xF8F13FD1, 0x345B3F4F, 0xBAD438AC, 0x767E3832, 0xAF5E2A9E, 0x63F42A00, 0xED7B2DE3, + 0x21D12D7D, 0x2B142464, 0xE7BE24FA, 0x69312319, 0xA59B2387, 0xF9766256, 0x35DC62C8, 0xBB53652B, 0x77F965B5, + 0x7D3C6CAC, 0xB1966C32, 0x3F196BD1, 0xF3B36B4F, 0x2A9379E3, 0xE639797D, 0x68B67E9E, 0xA41C7E00, 0xAED97719, + 0x62737787, 0xECFC7064, 0x205670FA, 0x85CD537D, 0x496753E3, 0xC7E85400, 0x0B42549E, 0x01875D87, 0xCD2D5D19, + 0x43A25AFA, 0x8F085A64, 0x562848C8, 0x9A824856, 0x140D4FB5, 0xD8A74F2B, 0xD2624632, 0x1EC846AC, 0x9047414F, + 0x5CED41D1, 0x299DC2ED, 0xE537C273, 0x6BB8C590, 0xA712C50E, 0xADD7CC17, 0x617DCC89, 0xEFF2CB6A, 0x2358CBF4, + 0xFA78D958, 0x36D2D9C6, 0xB85DDE25, 0x74F7DEBB, 0x7E32D7A2, 0xB298D73C, 0x3C17D0DF, 0xF0BDD041, 0x5526F3C6, + 0x998CF358, 0x1703F4BB, 0xDBA9F425, 0xD16CFD3C, 0x1DC6FDA2, 0x9349FA41, 0x5FE3FADF, 0x86C3E873, 0x4A69E8ED, + 0xC4E6EF0E, 0x084CEF90, 0x0289E689, 0xCE23E617, 0x40ACE1F4, 0x8C06E16A, 0xD0EBA0BB, 0x1C41A025, 0x92CEA7C6, + 0x5E64A758, 0x54A1AE41, 0x980BAEDF, 0x1684A93C, 0xDA2EA9A2, 0x030EBB0E, 0xCFA4BB90, 0x412BBC73, 0x8D81BCED, + 0x8744B5F4, 0x4BEEB56A, 0xC561B289, 0x09CBB217, 0xAC509190, 0x60FA910E, 0xEE7596ED, 0x22DF9673, 0x281A9F6A, + 0xE4B09FF4, 0x6A3F9817, 0xA6959889, 0x7FB58A25, 0xB31F8ABB, 0x3D908D58, 0xF13A8DC6, 0xFBFF84DF, 0x37558441, + 0xB9DA83A2, 0x7570833C, 0x533B85DA, 0x9F918544, 0x111E82A7, 0xDDB48239, 0xD7718B20, 0x1BDB8BBE, 0x95548C5D, + 0x59FE8CC3, 0x80DE9E6F, 0x4C749EF1, 0xC2FB9912, 0x0E51998C, 0x04949095, 0xC83E900B, 0x46B197E8, 0x8A1B9776, + 0x2F80B4F1, 0xE32AB46F, 0x6DA5B38C, 0xA10FB312, 0xABCABA0B, 0x6760BA95, 0xE9EFBD76, 0x2545BDE8, 0xFC65AF44, + 0x30CFAFDA, 0xBE40A839, 0x72EAA8A7, 0x782FA1BE, 0xB485A120, 0x3A0AA6C3, 0xF6A0A65D, 0xAA4DE78C, 0x66E7E712, + 0xE868E0F1, 0x24C2E06F, 0x2E07E976, 0xE2ADE9E8, 0x6C22EE0B, 0xA088EE95, 0x79A8FC39, 0xB502FCA7, 0x3B8DFB44, + 0xF727FBDA, 0xFDE2F2C3, 0x3148F25D, 0xBFC7F5BE, 0x736DF520, 0xD6F6D6A7, 0x1A5CD639, 0x94D3D1DA, 0x5879D144, + 0x52BCD85D, 0x9E16D8C3, 0x1099DF20, 0xDC33DFBE, 0x0513CD12, 0xC9B9CD8C, 0x4736CA6F, 0x8B9CCAF1, 0x8159C3E8, + 0x4DF3C376, 0xC37CC495, 0x0FD6C40B, 0x7AA64737, 0xB60C47A9, 0x3883404A, 0xF42940D4, 0xFEEC49CD, 0x32464953, + 0xBCC94EB0, 0x70634E2E, 0xA9435C82, 0x65E95C1C, 0xEB665BFF, 0x27CC5B61, 0x2D095278, 0xE1A352E6, 0x6F2C5505, + 0xA386559B, 0x061D761C, 0xCAB77682, 0x44387161, 0x889271FF, 0x825778E6, 0x4EFD7878, 0xC0727F9B, 0x0CD87F05, + 0xD5F86DA9, 0x19526D37, 0x97DD6AD4, 0x5B776A4A, 0x51B26353, 0x9D1863CD, 0x1397642E, 0xDF3D64B0, 0x83D02561, + 0x4F7A25FF, 0xC1F5221C, 0x0D5F2282, 0x079A2B9B, 0xCB302B05, 0x45BF2CE6, 0x89152C78, 0x50353ED4, 0x9C9F3E4A, + 0x121039A9, 0xDEBA3937, 0xD47F302E, 0x18D530B0, 0x965A3753, 0x5AF037CD, 0xFF6B144A, 0x33C114D4, 0xBD4E1337, + 0x71E413A9, 0x7B211AB0, 0xB78B1A2E, 0x39041DCD, 0xF5AE1D53, 0x2C8E0FFF, 0xE0240F61, 0x6EAB0882, 0xA201081C, + 0xA8C40105, 0x646E019B, 0xEAE10678, 0x264B06E6 }; +} diff --git a/qmq-common/src/main/java/qunar/tc/qmq/utils/DelayUtil.java b/qmq-common/src/main/java/qunar/tc/qmq/utils/DelayUtil.java new file mode 100644 index 00000000..7e56ca7c --- /dev/null +++ b/qmq-common/src/main/java/qunar/tc/qmq/utils/DelayUtil.java @@ -0,0 +1,34 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.utils; + +import qunar.tc.qmq.Message; + +import java.util.Date; + +/** + * Created by zhaohui.yu + * 6/14/18 + */ +public class DelayUtil { + private static final long MIN_DELAY_TIME = 500; + + public static boolean isDelayMessage(Message message) { + Date receiveTime = message.getScheduleReceiveTime(); + return receiveTime != null && (receiveTime.getTime() - System.currentTimeMillis()) >= MIN_DELAY_TIME; + } +} diff --git a/qmq-common/src/main/java/qunar/tc/qmq/utils/Flags.java b/qmq-common/src/main/java/qunar/tc/qmq/utils/Flags.java new file mode 100644 index 00000000..c79d2ef7 --- /dev/null +++ b/qmq-common/src/main/java/qunar/tc/qmq/utils/Flags.java @@ -0,0 +1,39 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.utils; + +/** + * Created by zhaohui.yu + * 8/22/18 + */ +public class Flags { + public static byte setDelay(byte flag, boolean isDelay) { + return !isDelay ? flag : (byte) (flag | 2); + } + + public static byte setTags(byte flag, boolean hasTag) { + return hasTag ? (byte) (flag | 4) : flag; + } + + public static boolean isDelay(byte flag) { + return (flag & 2) == 2; + } + + public static boolean hasTags(byte flag) { + return (flag & 4) == 4; + } +} diff --git a/qmq-common/src/main/java/qunar/tc/qmq/utils/ListUtils.java b/qmq-common/src/main/java/qunar/tc/qmq/utils/ListUtils.java new file mode 100644 index 00000000..c7b3ca8d --- /dev/null +++ b/qmq-common/src/main/java/qunar/tc/qmq/utils/ListUtils.java @@ -0,0 +1,54 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.utils; + +import java.util.Arrays; +import java.util.List; + +/** + * @author: leix.xie + * @date: 2018/10/29 16:12 + * @describe: + */ +public class ListUtils { + public static boolean contains(List source, byte[] target) { + for (byte[] bytes : source) { + if (Arrays.equals(bytes, target)) { + return true; + } + } + return false; + } + + public static boolean containsAll(List source, List target) { + if (source.size() < target.size()) return false; + for (byte[] bytes : target) + if (!contains(source, bytes)) { + return false; + } + return true; + } + + public static boolean intersection(List source, List target) { + for (byte[] bytes : target) { + if (contains(source, bytes)) { + return true; + } + } + return false; + } +} diff --git a/qmq-common/src/main/java/qunar/tc/qmq/utils/NetworkUtils.java b/qmq-common/src/main/java/qunar/tc/qmq/utils/NetworkUtils.java new file mode 100644 index 00000000..54433e84 --- /dev/null +++ b/qmq-common/src/main/java/qunar/tc/qmq/utils/NetworkUtils.java @@ -0,0 +1,97 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.utils; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.net.*; +import java.util.ArrayList; +import java.util.Enumeration; + +/** + * @author keli.wang + * @since 2017/9/1 + */ +public class NetworkUtils { + private static final Logger LOG = LoggerFactory.getLogger(NetworkUtils.class); + + public static String getLocalHostname() { + try { + return InetAddress.getLocalHost().getHostName(); + } catch (UnknownHostException e) { + LOG.error("get local hostname failed. return local ip instead.", e); + return getLocalAddress(); + } + } + + public static String getLocalAddress() { + try { + final Enumeration interfaces = NetworkInterface.getNetworkInterfaces(); + final ArrayList ipv4Result = new ArrayList<>(); + final ArrayList ipv6Result = new ArrayList<>(); + while (interfaces.hasMoreElements()) { + final NetworkInterface networkInterface = interfaces.nextElement(); + if (networkInterface.getDisplayName().contains("docker")) { + continue; + } + + final Enumeration addresses = networkInterface.getInetAddresses(); + while (addresses.hasMoreElements()) { + final InetAddress address = addresses.nextElement(); + if (!address.isLoopbackAddress()) { + if (address instanceof Inet6Address) { + ipv6Result.add(address.getHostAddress()); + } else { + ipv4Result.add(address.getHostAddress()); + } + } + } + } + + if (!ipv4Result.isEmpty()) { + for (String ip : ipv4Result) { + if (ip.startsWith("127.0") || ip.startsWith("192.168")) { + continue; + } + + return ip; + } + + return ipv4Result.get(ipv4Result.size() - 1); + } else if (!ipv6Result.isEmpty()) { + return ipv6Result.get(0); + } + + return InetAddress.getLocalHost().getHostAddress(); + } catch (Exception e) { + LOG.error("get local address failed", e); + } + + return null; + } + + public static boolean isValid(String address) { + try { + final String[] s = address.split(":"); + new InetSocketAddress(s[0], Integer.parseInt(s[1])); + return true; + } catch (Exception e) { + return false; + } + } +} diff --git a/qmq-common/src/main/java/qunar/tc/qmq/utils/ObjectUtils.java b/qmq-common/src/main/java/qunar/tc/qmq/utils/ObjectUtils.java new file mode 100644 index 00000000..08a267e6 --- /dev/null +++ b/qmq-common/src/main/java/qunar/tc/qmq/utils/ObjectUtils.java @@ -0,0 +1,31 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.utils; + +/** + * @author keli.wang + * @since 2018-11-27 + */ +public final class ObjectUtils { + public static T defaultIfNull(T cond, T defaultValue) { + if (cond == null) { + return defaultValue; + } else { + return cond; + } + } +} diff --git a/qmq-common/src/main/java/qunar/tc/qmq/utils/PayloadHolderUtils.java b/qmq-common/src/main/java/qunar/tc/qmq/utils/PayloadHolderUtils.java new file mode 100644 index 00000000..933e05af --- /dev/null +++ b/qmq-common/src/main/java/qunar/tc/qmq/utils/PayloadHolderUtils.java @@ -0,0 +1,101 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.utils; + +import io.netty.buffer.ByteBuf; + +import java.nio.ByteBuffer; +import java.util.HashMap; +import java.util.Map; + +/** + * @author yiqun.fan create on 17-8-2. + */ +public final class PayloadHolderUtils { + + public static void writeString(String s, ByteBuf out) { + byte[] bs = CharsetUtils.toUTF8Bytes(s); + out.writeShort((short) bs.length); + out.writeBytes(bs); + } + + public static String readString(ByteBuf in) { + int len = in.readShort(); + byte[] bs = new byte[len]; + in.readBytes(bs); + return CharsetUtils.toUTF8String(bs); + } + + public static void writeString(String s, ByteBuffer out) { + byte[] bs = CharsetUtils.toUTF8Bytes(s); + out.putShort((short) bs.length); + out.put(bs); + } + + public static String readString(ByteBuffer in) { + int len = in.getShort(); + byte[] bs = new byte[len]; + in.get(bs); + return CharsetUtils.toUTF8String(bs); + } + + public static void writeBytes(byte[] bs, ByteBuf out) { + out.writeInt(bs.length); + out.writeBytes(bs); + } + + public static byte[] readBytes(ByteBuf in) { + int len = in.readInt(); + byte[] bs = new byte[len]; + in.readBytes(bs); + return bs; + } + + public static byte[] readBytes(ByteBuffer in) { + int len = in.getInt(); + byte[] bs = new byte[len]; + in.get(bs); + return bs; + } + + public static void writeStringMap(Map map, ByteBuf out) { + if (map == null || map.isEmpty()) { + out.writeShort(0); + } else { + if (map.size() > Short.MAX_VALUE) { + throw new IndexOutOfBoundsException("map is too large. size=" + map.size()); + } + out.writeShort(map.size()); + for (Map.Entry entry : map.entrySet()) { + writeString(entry.getKey(), out); + writeString(entry.getValue(), out); + } + } + } + + public static Map readStringHashMap(ByteBuf in) { + return readStringMap(in, new HashMap()); + } + + public static Map readStringMap(ByteBuf in, Map map) { + short size = in.readShort(); + for (int i = 0; i < size; i++) { + map.put(readString(in), readString(in)); + } + return map; + } +} diff --git a/qmq-common/src/main/java/qunar/tc/qmq/utils/PidUtil.java b/qmq-common/src/main/java/qunar/tc/qmq/utils/PidUtil.java new file mode 100644 index 00000000..06fe48e6 --- /dev/null +++ b/qmq-common/src/main/java/qunar/tc/qmq/utils/PidUtil.java @@ -0,0 +1,32 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.utils; + +import java.lang.management.ManagementFactory; +import java.lang.management.RuntimeMXBean; + +public class PidUtil { + public static int getPid() { + try { + RuntimeMXBean runtime = ManagementFactory.getRuntimeMXBean(); + String name = runtime.getName(); // format: "pid@hostname" + return Integer.parseInt(name.substring(0, name.indexOf('@'))); + } catch (Throwable e) { + return 0; + } + } +} diff --git a/qmq-common/src/main/java/qunar/tc/qmq/utils/RetrySubjectUtils.java b/qmq-common/src/main/java/qunar/tc/qmq/utils/RetrySubjectUtils.java new file mode 100644 index 00000000..82a2c528 --- /dev/null +++ b/qmq-common/src/main/java/qunar/tc/qmq/utils/RetrySubjectUtils.java @@ -0,0 +1,91 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.utils; + +import com.google.common.base.Joiner; +import com.google.common.base.Optional; +import com.google.common.base.Splitter; +import com.google.common.base.Strings; + +import java.util.List; + +/** + * @author keli.wang + * @since 2017/8/23 + */ +public final class RetrySubjectUtils { + private static final Joiner RETRY_SUBJECT_JOINER = Joiner.on('%'); + private static final Splitter RETRY_SUBJECT_SPLITTER = Splitter.on('%').trimResults().omitEmptyStrings(); + private static final String RETRY_SUBJECT_PREFIX = "%RETRY"; + private static final String DEAD_RETRY_SUBJECT_PREFIX = "%DEAD_RETRY"; + + private RetrySubjectUtils() { + } + + public static boolean isRealSubject(final String subject) { + return !Strings.isNullOrEmpty(subject) && !isRetrySubject(subject) && !isDeadRetrySubject(subject); + } + + public static String buildRetrySubject(final String subject, final String group) { + return RETRY_SUBJECT_JOINER.join(RETRY_SUBJECT_PREFIX, subject, group); + } + + public static boolean isRetrySubject(final String subject) { + return Strings.nullToEmpty(subject).startsWith(RETRY_SUBJECT_PREFIX); + } + + public static String buildDeadRetrySubject(final String subject, final String group) { + return RETRY_SUBJECT_JOINER.join(DEAD_RETRY_SUBJECT_PREFIX, subject, group); + } + + public static boolean isDeadRetrySubject(final String subject) { + return Strings.nullToEmpty(subject).startsWith(DEAD_RETRY_SUBJECT_PREFIX); + } + + public static String getRealSubject(final String subject) { + final Optional optional = getSubject(subject); + if (optional.isPresent()) { + return optional.get(); + } + return subject; + } + + public static Optional getSubject(final String retrySubject) { + if (!isRetrySubject(retrySubject) && !isDeadRetrySubject(retrySubject)) { + return Optional.absent(); + } + final List parts = RETRY_SUBJECT_SPLITTER.splitToList(retrySubject); + if (parts.size() != 3) { + return Optional.absent(); + } else { + return Optional.of(parts.get(1)); + } + } + + public static String[] parseSubjectAndGroup(String subject) { + if (!isRetrySubject(subject) && !isDeadRetrySubject(subject)) { + return null; + } + + final List parts = RETRY_SUBJECT_SPLITTER.splitToList(subject); + if (parts.size() != 3) { + return null; + } else { + return new String[]{parts.get(1), parts.get(2)}; + } + } +} diff --git a/qmq-common/src/main/resources/META-INF/services/qunar.tc.qmq.configuration.DynamicConfigFactory b/qmq-common/src/main/resources/META-INF/services/qunar.tc.qmq.configuration.DynamicConfigFactory new file mode 100644 index 00000000..3672445f --- /dev/null +++ b/qmq-common/src/main/resources/META-INF/services/qunar.tc.qmq.configuration.DynamicConfigFactory @@ -0,0 +1 @@ +qunar.tc.qmq.configuration.local.LocalDynamicConfigFactory \ No newline at end of file diff --git a/qmq-delay-server/pom.xml b/qmq-delay-server/pom.xml new file mode 100644 index 00000000..ad880cd8 --- /dev/null +++ b/qmq-delay-server/pom.xml @@ -0,0 +1,39 @@ + + + 4.0.0 + + + qunar.tc + qmq + 4.0.30 + + + qmq-delay-server + jar + + + + ${project.groupId} + qmq-server-common + + + ${project.groupId} + qmq-sync + + + ${project.groupId} + qmq-store + + + ${project.groupId} + qmq-client + + + joda-time + joda-time + + + + \ No newline at end of file diff --git a/qmq-delay-server/src/main/java/qunar/tc/qmq/delay/DefaultDelayLogFacade.java b/qmq-delay-server/src/main/java/qunar/tc/qmq/delay/DefaultDelayLogFacade.java new file mode 100644 index 00000000..c3b03b23 --- /dev/null +++ b/qmq-delay-server/src/main/java/qunar/tc/qmq/delay/DefaultDelayLogFacade.java @@ -0,0 +1,198 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.delay; + +import io.netty.buffer.ByteBuf; +import qunar.tc.qmq.delay.base.LongHashSet; +import qunar.tc.qmq.delay.base.ReceivedDelayMessage; +import qunar.tc.qmq.delay.base.ReceivedResult; +import qunar.tc.qmq.delay.cleaner.LogCleaner; +import qunar.tc.qmq.delay.config.StoreConfiguration; +import qunar.tc.qmq.delay.store.IterateOffsetManager; +import qunar.tc.qmq.delay.store.log.*; +import qunar.tc.qmq.delay.store.model.AppendLogResult; +import qunar.tc.qmq.delay.store.model.LogRecord; +import qunar.tc.qmq.delay.store.model.RawMessageExtend; +import qunar.tc.qmq.delay.store.model.ScheduleSetRecord; +import qunar.tc.qmq.delay.store.visitor.LogVisitor; +import qunar.tc.qmq.delay.wheel.WheelLoadCursor; +import qunar.tc.qmq.store.SegmentBuffer; +import qunar.tc.qmq.sync.DelaySyncRequest; + +import java.nio.ByteBuffer; +import java.util.List; +import java.util.function.Consumer; +import java.util.function.Function; + +/** + * @author xufeng.deng dennisdxf@gmail.com + * @since 2018-07-20 10:20 + */ +public class DefaultDelayLogFacade implements DelayLogFacade { + private final IterateOffsetManager offsetManager; + private final ScheduleLog scheduleLog; + private final DispatchLog dispatchLog; + private final MessageLog messageLog; + private final LogFlusher logFlusher; + private final LogCleaner cleaner; + private final MessageLogReplayer replayer; + + public DefaultDelayLogFacade(final StoreConfiguration config, final Function func) { + this.messageLog = new MessageLog(config); + this.scheduleLog = new ScheduleLog(config); + this.dispatchLog = new DispatchLog(config); + this.offsetManager = new IterateOffsetManager(config.getCheckpointStorePath(), scheduleLog::flush); + this.replayer = new MessageLogReplayer(this, func); + this.logFlusher = new LogFlusher(messageLog, offsetManager, dispatchLog); + this.cleaner = new LogCleaner(config, dispatchLog, scheduleLog, messageLog); + } + + @Override + public ReceivedResult appendMessageLog(final ReceivedDelayMessage message) { + final RawMessageExtend rawMessage = message.getMessage(); + final String msgId = rawMessage.getHeader().getMessageId(); + AppendLogResult result = messageLog.append(rawMessage); + return new ReceivedResult(msgId, result.getCode(), result.getRemark(), result.getAdditional().getMessageOffset()); + } + + @Override + public long getMessageLogMinOffset() { + return messageLog.getMinOffset(); + } + + @Override + public long getMessageLogMaxOffset() { + return messageLog.getMaxOffset(); + } + + @Override + public long getDispatchLogMaxOffset(final int dispatchSegmentBaseOffset) { + return dispatchLog.getMaxOffset(dispatchSegmentBaseOffset); + } + + @Override + public DelaySyncRequest.DispatchLogSyncRequest getDispatchLogSyncMaxRequest() { + return dispatchLog.getSyncMaxRequest(); + } + + @Override + public boolean appendMessageLogData(final long startOffset, final ByteBuffer buffer) { + return messageLog.appendData(startOffset, buffer); + } + + @Override + public boolean appendDispatchLogData(final long startOffset, final int baseOffset, final ByteBuffer body) { + return dispatchLog.appendData(startOffset, baseOffset, body); + } + + @Override + public SegmentBuffer getMessageLogs(final long startSyncOffset) { + return messageLog.getMessageLogData(startSyncOffset); + } + + @Override + public SegmentBuffer getDispatchLogs(final int segmentBaseOffset, final long dispatchLogOffset) { + return dispatchLog.getDispatchLogData(segmentBaseOffset, dispatchLogOffset); + } + + @Override + public void start() { + logFlusher.start(); + replayer.start(); + cleaner.start(); + } + + @Override + public void shutdown() { + cleaner.shutdown(); + replayer.shutdown(); + logFlusher.shutdown(); + scheduleLog.destroy(); + } + + @Override + public List recoverLogRecord(final List pureRecords) { + return scheduleLog.recoverLogRecord(pureRecords); + } + + @Override + public void appendDispatchLog(LogRecord record) { + dispatchLog.append(record); + } + + @Override + public DispatchLogSegment latestDispatchSegment() { + return dispatchLog.latestSegment(); + } + + @Override + public DispatchLogSegment lowerDispatchSegment(final int baseOffset) { + return dispatchLog.lowerSegment(baseOffset); + } + + @Override + public ScheduleSetSegment loadScheduleLogSegment(final int segmentBaseOffset) { + return scheduleLog.loadSegment(segmentBaseOffset); + } + + @Override + public WheelLoadCursor.Cursor loadUnDispatch(final ScheduleSetSegment setSegment, final LongHashSet dispatchedSet, final Consumer refresh) { + return scheduleLog.loadUnDispatch(setSegment, dispatchedSet, refresh); + } + + @Override + public int higherScheduleBaseOffset(int index) { + return scheduleLog.higherBaseOffset(index); + } + + @Override + public int higherDispatchLogBaseOffset(int segmentBaseOffset) { + return dispatchLog.higherBaseOffset(segmentBaseOffset); + } + + @Override + public LogVisitor newMessageLogVisitor(long start) { + return messageLog.newVisitor(start); + } + + @Override + public AppendLogResult appendScheduleLog(LogRecord event) { + return scheduleLog.append(event); + } + + @Override + public long initialMessageIterateFrom() { + long iterateOffset = offsetManager.getIterateOffset(); + if (iterateOffset <= 0) { + return getMessageLogMaxOffset(); + } + if (iterateOffset > getMessageLogMaxOffset()) { + return getMessageLogMaxOffset(); + } + return iterateOffset; + } + + @Override + public void updateIterateOffset(long checkpoint) { + offsetManager.updateIterateOffset(checkpoint); + } + + @Override + public void blockUntilReplayDone() { + replayer.blockUntilReplayDone(); + } +} diff --git a/qmq-delay-server/src/main/java/qunar/tc/qmq/delay/DelayLogFacade.java b/qmq-delay-server/src/main/java/qunar/tc/qmq/delay/DelayLogFacade.java new file mode 100644 index 00000000..c4a22cc1 --- /dev/null +++ b/qmq-delay-server/src/main/java/qunar/tc/qmq/delay/DelayLogFacade.java @@ -0,0 +1,89 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.delay; + +import io.netty.buffer.ByteBuf; +import qunar.tc.qmq.delay.base.LongHashSet; +import qunar.tc.qmq.delay.base.ReceivedDelayMessage; +import qunar.tc.qmq.delay.base.ReceivedResult; +import qunar.tc.qmq.delay.store.log.DispatchLogSegment; +import qunar.tc.qmq.delay.store.log.ScheduleSetSegment; +import qunar.tc.qmq.delay.store.model.AppendLogResult; +import qunar.tc.qmq.delay.store.model.LogRecord; +import qunar.tc.qmq.delay.store.model.ScheduleSetRecord; +import qunar.tc.qmq.delay.store.visitor.LogVisitor; +import qunar.tc.qmq.delay.wheel.WheelLoadCursor; +import qunar.tc.qmq.store.SegmentBuffer; +import qunar.tc.qmq.sync.DelaySyncRequest; + +import java.nio.ByteBuffer; +import java.util.List; +import java.util.function.Consumer; + +/** + * @author xufeng.deng dennisdxf@gmail.com + * @since 2018-07-17 16:57 + */ +public interface DelayLogFacade { + void start(); + + void shutdown(); + + ReceivedResult appendMessageLog(ReceivedDelayMessage message); + + long getMessageLogMinOffset(); + + SegmentBuffer getMessageLogs(long startSyncOffset); + + SegmentBuffer getDispatchLogs(int segmentBaseOffset, long dispatchLogOffset); + + long getMessageLogMaxOffset(); + + long getDispatchLogMaxOffset(int dispatchSegmentBaseOffset); + + DelaySyncRequest.DispatchLogSyncRequest getDispatchLogSyncMaxRequest(); + + boolean appendMessageLogData(long startOffset, ByteBuffer buffer); + + boolean appendDispatchLogData(long startOffset, int baseOffset, ByteBuffer body); + + List recoverLogRecord(List pureRecords); + + void appendDispatchLog(LogRecord record); + + DispatchLogSegment latestDispatchSegment(); + + DispatchLogSegment lowerDispatchSegment(int latestOffset); + + ScheduleSetSegment loadScheduleLogSegment(int segmentBaseOffset); + + WheelLoadCursor.Cursor loadUnDispatch(ScheduleSetSegment setSegment, LongHashSet dispatchedSet, Consumer refresh); + + int higherScheduleBaseOffset(int index); + + LogVisitor newMessageLogVisitor(long start); + + AppendLogResult appendScheduleLog(LogRecord event); + + long initialMessageIterateFrom(); + + void updateIterateOffset(long checkpoint); + + void blockUntilReplayDone(); + + int higherDispatchLogBaseOffset(int segmentBaseOffset); +} diff --git a/qmq-delay-server/src/main/java/qunar/tc/qmq/delay/EventListener.java b/qmq-delay-server/src/main/java/qunar/tc/qmq/delay/EventListener.java new file mode 100644 index 00000000..49b55c93 --- /dev/null +++ b/qmq-delay-server/src/main/java/qunar/tc/qmq/delay/EventListener.java @@ -0,0 +1,27 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.delay; + +/** + * @author xufeng.deng dennisdxf@gmail.com + * @since 2018-07-13 9:42 + */ +public interface EventListener { + + void post(T event); + +} diff --git a/qmq-delay-server/src/main/java/qunar/tc/qmq/delay/LogFlusher.java b/qmq-delay-server/src/main/java/qunar/tc/qmq/delay/LogFlusher.java new file mode 100644 index 00000000..da3f9861 --- /dev/null +++ b/qmq-delay-server/src/main/java/qunar/tc/qmq/delay/LogFlusher.java @@ -0,0 +1,53 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.delay; + +import qunar.tc.qmq.delay.store.IterateOffsetManager; +import qunar.tc.qmq.delay.store.log.DispatchLog; +import qunar.tc.qmq.delay.store.log.MessageLog; +import qunar.tc.qmq.store.PeriodicFlushService; + +/** + * @author xufeng.deng dennisdxf@gmail.com + * @since 2018-07-19 17:14 + */ +public class LogFlusher implements Switchable { + private final PeriodicFlushService messageLogFlushService; + private final PeriodicFlushService dispatchLogFlushService; + private final PeriodicFlushService iterateOffsetFlushService; + + LogFlusher(MessageLog messageLog, IterateOffsetManager offsetManager, DispatchLog dispatchLog) { + this.messageLogFlushService = new PeriodicFlushService(messageLog.getProvider()); + this.dispatchLogFlushService = new PeriodicFlushService(dispatchLog.getProvider()); + this.iterateOffsetFlushService = new PeriodicFlushService(offsetManager.getFlushProvider()); + } + + @Override + public void start() { + messageLogFlushService.start(); + dispatchLogFlushService.start(); + iterateOffsetFlushService.start(); + } + + @Override + public void shutdown() { + messageLogFlushService.close(); + dispatchLogFlushService.close(); + iterateOffsetFlushService.close(); + } + +} diff --git a/qmq-delay-server/src/main/java/qunar/tc/qmq/delay/MessageIterateEventListener.java b/qmq-delay-server/src/main/java/qunar/tc/qmq/delay/MessageIterateEventListener.java new file mode 100644 index 00000000..14a5febf --- /dev/null +++ b/qmq-delay-server/src/main/java/qunar/tc/qmq/delay/MessageIterateEventListener.java @@ -0,0 +1,65 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.delay; + +import io.netty.buffer.ByteBuf; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import qunar.tc.qmq.delay.base.AppendException; +import qunar.tc.qmq.delay.meta.BrokerRoleManager; +import qunar.tc.qmq.delay.store.model.AppendLogResult; +import qunar.tc.qmq.delay.store.model.LogRecord; +import qunar.tc.qmq.protocol.producer.MessageProducerCode; + +import java.util.function.Function; + +/** + * @author xufeng.deng dennisdxf@gmail.com + * @since 2018-07-19 18:21 + */ +public class MessageIterateEventListener implements EventListener { + private static final Logger LOGGER = LoggerFactory.getLogger(MessageIterateEventListener.class); + + private final DelayLogFacade facade; + private final Function iterateCallback; + + MessageIterateEventListener(final DelayLogFacade facade, Function iterateCallback) { + this.facade = facade; + this.iterateCallback = iterateCallback; + } + + @Override + public void post(LogRecord event) { + AppendLogResult result = facade.appendScheduleLog(event); + int code = result.getCode(); + if (MessageProducerCode.SUCCESS != code) { + LOGGER.error("appendMessageLog schedule log error,log:{} {},code:{}", event.getSubject(), event.getMessageId(), code); + throw new AppendException("appendScheduleLogError"); + } + + if (BrokerRoleManager.isDelayMaster()) { + process(result.getAdditional()); + } else { + ScheduleIndex.release(result.getAdditional()); + } + } + + private void process(ByteBuf record) { + if (iterateCallback != null && iterateCallback.apply(record)) return; + ScheduleIndex.release(record); + } +} diff --git a/qmq-delay-server/src/main/java/qunar/tc/qmq/delay/MessageLogReplayer.java b/qmq-delay-server/src/main/java/qunar/tc/qmq/delay/MessageLogReplayer.java new file mode 100644 index 00000000..69af3a55 --- /dev/null +++ b/qmq-delay-server/src/main/java/qunar/tc/qmq/delay/MessageLogReplayer.java @@ -0,0 +1,164 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.delay; + +import com.google.common.util.concurrent.RateLimiter; +import io.netty.buffer.ByteBuf; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import qunar.tc.qmq.delay.store.model.LogRecord; +import qunar.tc.qmq.delay.store.visitor.DelayMessageLogVisitor; +import qunar.tc.qmq.delay.store.visitor.LogVisitor; +import qunar.tc.qmq.metrics.Metrics; + +import java.util.Optional; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.atomic.LongAdder; +import java.util.function.Function; + +import static qunar.tc.qmq.store.MessageLog.PER_SEGMENT_FILE_SIZE; + +/** + * @author xufeng.deng dennisdxf@gmail.com + * @since 2018-07-19 17:43 + */ +public class MessageLogReplayer implements Switchable { + private static final Logger LOGGER = LoggerFactory.getLogger(MessageLogReplayer.class); + private static final RateLimiter LOG_LIMITER = RateLimiter.create((double) 1 / 6); + private final DelayLogFacade facade; + private final Thread dispatcherThread; + private final LongAdder iterateFrom; + private final EventListener dispatcher; + private volatile boolean stop = true; + + MessageLogReplayer(final DelayLogFacade facade, final Function func) { + this.facade = facade; + this.dispatcher = new MessageIterateEventListener(facade, func); + this.iterateFrom = new LongAdder(); + this.iterateFrom.add(facade.initialMessageIterateFrom()); + this.dispatcherThread = new Thread(new Dispatcher(iterateFrom.longValue())); + + Metrics.gauge("ReplayMessageLogLag", () -> (double) replayMessageLogLag()); + } + + private long replayMessageLogLag() { + return facade.getMessageLogMaxOffset() - iterateFrom.longValue(); + } + + @Override + public void start() { + stop = false; + dispatcherThread.start(); + } + + @Override + public void shutdown() { + stop = true; + try { + dispatcherThread.join(); + } catch (InterruptedException e) { + LOGGER.error("message log replay error,iterate form:{}", iterateFrom.longValue(), e); + } + } + + void blockUntilReplayDone() { + LOGGER.info("replay message log initial lag: {}; min: {}, max: {}, from: {}", + replayMessageLogLag(), facade.getMessageLogMinOffset(), facade.getMessageLogMaxOffset(), iterateFrom.longValue()); + + while (replayMessageLogLag() > 0) { + LOGGER.info("waiting replay message log ..."); + try { + TimeUnit.SECONDS.sleep(1); + } catch (InterruptedException e) { + LOGGER.warn("block until replay done interrupted", e); + } + } + } + + private class Dispatcher implements Runnable { + private final AtomicLong cursor; + + Dispatcher(long iterate) { + this.cursor = new AtomicLong(iterate); + } + + @Override + public void run() { + while (!stop) { + try { + processLog(cursor.get()); + } catch (Throwable e) { + Metrics.counter("ReplayMessageLogFailed").inc(); + if (LOG_LIMITER.tryAcquire()) { + LOGGER.error("replay message log failed, will retry.cursor:{} iterateOffset:{}", cursor.get(), iterateFrom.longValue(), e); + } + } + } + } + + private void processLog(long cursor) { + long iterate = iterateFrom.longValue(); + if (cursor < iterate) { + if (LOG_LIMITER.tryAcquire()) { + LOGGER.info("replay message log failed,cursor < iterate,cursor:{},iterate:{}", cursor, iterate); + } + if ((iterate % PER_SEGMENT_FILE_SIZE) != 0) throw new RuntimeException("MessageReplayLessThanEx"); + this.cursor.set(iterate); + } + + if (cursor > iterate) { + LOGGER.error("replay message log happened accident,cursor:{},iterate:{}", cursor, iterate); + iterateFrom.add(cursor - iterate); + } + + final LogVisitor visitor = facade.newMessageLogVisitor(iterateFrom.longValue()); + adjustOffset(visitor); + + while (true) { + final Optional recordOptional = visitor.nextRecord(); + if (recordOptional.isPresent() + && recordOptional.get() instanceof DelayMessageLogVisitor.EmptyLogRecord) { + break; + } + + recordOptional.ifPresent((record) -> { + dispatcher.post(record); + long checkpoint = record.getStartWroteOffset() + record.getRecordSize(); + this.cursor.addAndGet(record.getRecordSize()); + facade.updateIterateOffset(checkpoint); + }); + } + iterateFrom.add(visitor.visitedBufferSize()); + + try { + TimeUnit.MILLISECONDS.sleep(5); + } catch (InterruptedException e) { + LOGGER.warn("message log iterate sleep interrupted"); + } + } + + private void adjustOffset(final LogVisitor visitor) { + long startOffset = ((DelayMessageLogVisitor) visitor).startOffset(); + long iterateLongValue = iterateFrom.longValue(); + if (startOffset > iterateLongValue) { + iterateFrom.add(startOffset - iterateLongValue); + this.cursor.set(startOffset); + } + } + } +} diff --git a/qmq-delay-server/src/main/java/qunar/tc/qmq/delay/ScheduleIndex.java b/qmq-delay-server/src/main/java/qunar/tc/qmq/delay/ScheduleIndex.java new file mode 100644 index 00000000..a509c708 --- /dev/null +++ b/qmq-delay-server/src/main/java/qunar/tc/qmq/delay/ScheduleIndex.java @@ -0,0 +1,60 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.delay; + +import io.netty.buffer.ByteBuf; +import io.netty.buffer.PooledByteBufAllocator; +import io.netty.util.ReferenceCountUtil; + +import java.util.List; + +public class ScheduleIndex { + public static ByteBuf buildIndex(long scheduleTime, long offset, int size, long sequence) { + ByteBuf index = PooledByteBufAllocator.DEFAULT.ioBuffer(8 + 8 + 4 + 8); + index.writeLong(scheduleTime); + index.writeLong(offset); + index.writeInt(size); + index.writeLong(sequence); + return index; + } + + public static long scheduleTime(ByteBuf index) { + return index.getLong(0); + } + + public static long offset(ByteBuf index) { + return index.getLong(8); + } + + public static int size(ByteBuf index) { + return index.getInt(16); + } + + public static long sequence(ByteBuf index) { + return index.getLong(20); + } + + public static void release(List resources) { + for (ByteBuf resource : resources) { + release(resource); + } + } + + public static void release(ByteBuf resource) { + ReferenceCountUtil.safeRelease(resource); + } +} diff --git a/qmq-delay-server/src/main/java/qunar/tc/qmq/delay/Switchable.java b/qmq-delay-server/src/main/java/qunar/tc/qmq/delay/Switchable.java new file mode 100644 index 00000000..87799f93 --- /dev/null +++ b/qmq-delay-server/src/main/java/qunar/tc/qmq/delay/Switchable.java @@ -0,0 +1,29 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.delay; + +/** + * @author xufeng.deng dennisdxf@gmail.com + * @since 2018-07-16 16:45 + */ +public interface Switchable { + + void start(); + + void shutdown(); + +} diff --git a/qmq-delay-server/src/main/java/qunar/tc/qmq/delay/base/AppendException.java b/qmq-delay-server/src/main/java/qunar/tc/qmq/delay/base/AppendException.java new file mode 100644 index 00000000..8998f7f7 --- /dev/null +++ b/qmq-delay-server/src/main/java/qunar/tc/qmq/delay/base/AppendException.java @@ -0,0 +1,28 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.delay.base; + +/** + * @author xufeng.deng dennisdxf@gmail.com + * @since 2018-08-03 11:01 + */ +public class AppendException extends RuntimeException { + + public AppendException(String message) { + super(message); + } +} diff --git a/qmq-delay-server/src/main/java/qunar/tc/qmq/delay/base/GroupSendException.java b/qmq-delay-server/src/main/java/qunar/tc/qmq/delay/base/GroupSendException.java new file mode 100644 index 00000000..6e084350 --- /dev/null +++ b/qmq-delay-server/src/main/java/qunar/tc/qmq/delay/base/GroupSendException.java @@ -0,0 +1,27 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.delay.base; + +/** + * @author xufeng.deng dennisdxf@gmail.com + * @since 2018-09-03 10:27 + */ +public class GroupSendException extends RuntimeException { + public GroupSendException(Throwable cause) { + super(cause); + } +} diff --git a/qmq-delay-server/src/main/java/qunar/tc/qmq/delay/base/LongHashSet.java b/qmq-delay-server/src/main/java/qunar/tc/qmq/delay/base/LongHashSet.java new file mode 100644 index 00000000..43c28997 --- /dev/null +++ b/qmq-delay-server/src/main/java/qunar/tc/qmq/delay/base/LongHashSet.java @@ -0,0 +1,140 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.delay.base; + +import java.util.Arrays; + +/** + * Created by zhaohui.yu + * 8/20/18 + */ +public class LongHashSet { + + static final int MISSING_VALUE = -1; + + private boolean containsMissingValue; + private final float loadFactor; + private int resizeThreshold; + private int sizeOfArrayValues; + + private long[] values; + + public LongHashSet(final int proposedCapacity) { + this.loadFactor = 0.55F; + sizeOfArrayValues = 0; + final int capacity = findNextPositivePowerOfTwo(proposedCapacity); + resizeThreshold = (int) (capacity * loadFactor); + values = new long[capacity]; + Arrays.fill(values, MISSING_VALUE); + } + + public boolean set(final long value) { + if (value == MISSING_VALUE) { + final boolean previousContainsMissingValue = this.containsMissingValue; + containsMissingValue = true; + return !previousContainsMissingValue; + } + + final long[] values = this.values; + final int mask = values.length - 1; + int index = hash(value, mask); + + while (values[index] != MISSING_VALUE) { + index = next(index, mask); + } + + values[index] = value; + sizeOfArrayValues++; + + if (sizeOfArrayValues > resizeThreshold) { + increaseCapacity(); + } + + return true; + } + + public boolean contains(final long value) { + if (value == MISSING_VALUE) { + return containsMissingValue; + } + + final long[] values = this.values; + final int mask = values.length - 1; + int index = hash(value, mask); + + while (values[index] != MISSING_VALUE) { + if (values[index] == value) { + return true; + } + + index = next(index, mask); + } + + return false; + } + + public static int findNextPositivePowerOfTwo(final int value) { + return 1 << (32 - Integer.numberOfLeadingZeros(value - 1)); + } + + public static int hash(final long value, final int mask) { + long hash = value * 31; + hash = (int) hash ^ (int) (hash >>> 32); + + return (int) hash & mask; + } + + private void increaseCapacity() { + final int newCapacity = values.length * 2; + if (newCapacity < 0) { + throw new IllegalStateException("max capacity reached at size=" + size()); + } + + rehash(newCapacity); + } + + private void rehash(final int newCapacity) { + final int capacity = newCapacity; + final int mask = newCapacity - 1; + resizeThreshold = (int) (newCapacity * loadFactor); + + final long[] tempValues = new long[capacity]; + Arrays.fill(tempValues, MISSING_VALUE); + + for (final long value : values) { + if (value != MISSING_VALUE) { + int newHash = hash(value, mask); + while (tempValues[newHash] != MISSING_VALUE) { + newHash = ++newHash & mask; + } + + tempValues[newHash] = value; + } + } + + values = tempValues; + } + + + private static int next(final int index, final int mask) { + return (index + 1) & mask; + } + + public int size() { + return sizeOfArrayValues + (containsMissingValue ? 1 : 0); + } +} \ No newline at end of file diff --git a/qmq-delay-server/src/main/java/qunar/tc/qmq/delay/base/ReceivedDelayMessage.java b/qmq-delay-server/src/main/java/qunar/tc/qmq/delay/base/ReceivedDelayMessage.java new file mode 100644 index 00000000..30d7185d --- /dev/null +++ b/qmq-delay-server/src/main/java/qunar/tc/qmq/delay/base/ReceivedDelayMessage.java @@ -0,0 +1,81 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.delay.base; + +import com.google.common.util.concurrent.SettableFuture; +import qunar.tc.qmq.delay.store.model.RawMessageExtend; + +/** + * @author xufeng.deng dennisdxf@gmail.com + * @since 2018-07-26 14:17 + */ +public class ReceivedDelayMessage { + private final RawMessageExtend message; + private final SettableFuture promise; + private final long receivedTime; + + public ReceivedDelayMessage(RawMessageExtend message, long receivedTime) { + this.message = message; + this.receivedTime = receivedTime; + this.promise = SettableFuture.create(); + } + + public RawMessageExtend getMessage() { + return message; + } + + public void done(ReceivedResult result) { + promise.set(result); + } + + public String getMessageId() { + return message.getHeader().getMessageId(); + } + + public String getSubject() { + return message.getHeader().getSubject(); + } + + public SettableFuture getPromise() { + return promise; + } + + public long getReceivedTime() { + return receivedTime; + } + + public boolean isExpired() { + return System.currentTimeMillis() > message.getHeader().getExpireTime(); + } + + public long getScheduleTime() { + return message.getScheduleTime(); + } + + public void adjustScheduleTime(long scheduleTime) { + message.setScheduleTime(scheduleTime); + } + + @Override + public String toString() { + return "ReceivedDelayMessage{" + + "message=" + message + + ", promise=" + promise + + ", receivedTime=" + receivedTime + + '}'; + } +} diff --git a/qmq-delay-server/src/main/java/qunar/tc/qmq/delay/base/ReceivedResult.java b/qmq-delay-server/src/main/java/qunar/tc/qmq/delay/base/ReceivedResult.java new file mode 100644 index 00000000..e9799106 --- /dev/null +++ b/qmq-delay-server/src/main/java/qunar/tc/qmq/delay/base/ReceivedResult.java @@ -0,0 +1,55 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.delay.base; + +/** + * @author xufeng.deng dennisdxf@gmail.com + * @since 2018-07-26 14:19 + */ +public class ReceivedResult { + private final String messageId; + private final int code; + private final String remark; + private final long messageLogOffset; + + public ReceivedResult(String messageId, int code, String remark) { + this(messageId, code, remark, -1); + } + + public ReceivedResult(String messageId, int code, String remark, long messageLogOffset) { + this.messageId = messageId; + this.code = code; + this.remark = remark; + this.messageLogOffset = messageLogOffset; + } + + public String getMessageId() { + return messageId; + } + + public int getCode() { + return code; + } + + public String getRemark() { + return remark; + } + + public long getMessageLogOffset() { + return messageLogOffset; + } +} diff --git a/qmq-delay-server/src/main/java/qunar/tc/qmq/delay/base/SegmentBufferExtend.java b/qmq-delay-server/src/main/java/qunar/tc/qmq/delay/base/SegmentBufferExtend.java new file mode 100644 index 00000000..d5b4992e --- /dev/null +++ b/qmq-delay-server/src/main/java/qunar/tc/qmq/delay/base/SegmentBufferExtend.java @@ -0,0 +1,39 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.delay.base; + +import qunar.tc.qmq.store.LogSegment; +import qunar.tc.qmq.store.SegmentBuffer; + +import java.nio.ByteBuffer; + +/** + * @author xufeng.deng dennisdxf@gmail.com + * @since 2018-08-13 14:32 + */ +public class SegmentBufferExtend extends SegmentBuffer { + private int baseOffset; + + public SegmentBufferExtend(long startOffset, ByteBuffer buffer, int size, int baseOffset, LogSegment logSegment) { + super(startOffset, buffer, size, logSegment); + this.baseOffset = baseOffset; + } + + public int getBaseOffset() { + return baseOffset; + } +} diff --git a/qmq-delay-server/src/main/java/qunar/tc/qmq/delay/cleaner/LogCleaner.java b/qmq-delay-server/src/main/java/qunar/tc/qmq/delay/cleaner/LogCleaner.java new file mode 100644 index 00000000..9fa17ba5 --- /dev/null +++ b/qmq-delay-server/src/main/java/qunar/tc/qmq/delay/cleaner/LogCleaner.java @@ -0,0 +1,96 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.delay.cleaner; + +import com.google.common.util.concurrent.ThreadFactoryBuilder; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import qunar.tc.qmq.delay.Switchable; +import qunar.tc.qmq.delay.config.StoreConfiguration; +import qunar.tc.qmq.delay.store.log.DispatchLog; +import qunar.tc.qmq.delay.store.log.MessageLog; +import qunar.tc.qmq.delay.store.log.ScheduleLog; + +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; + +/** + * @author xufeng.deng dennisdxf@gmail.com + * @since 2018-07-30 15:56 + */ +public class LogCleaner implements Switchable { + private static final Logger LOGGER = LoggerFactory.getLogger(LogCleaner.class); + + private final DispatchLog dispatchLog; + private final MessageLog messageLog; + private final ScheduleLog scheduleLog; + private final StoreConfiguration config; + private final ScheduledExecutorService cleanScheduler; + + public LogCleaner(StoreConfiguration config, DispatchLog dispatchLog, ScheduleLog scheduleLog, MessageLog messageLog) { + this.config = config; + this.scheduleLog = scheduleLog; + this.dispatchLog = dispatchLog; + this.messageLog = messageLog; + + this.cleanScheduler = Executors.newSingleThreadScheduledExecutor(new ThreadFactoryBuilder().setNameFormat("delay-broker-cleaner-%d").build()); + } + + private void cleanMessageLog() { + messageLog.clean(); + } + + private void cleanDispatchLog(CleanHook hook) { + dispatchLog.clean(hook); + } + + private void cleanScheduleOldLog() { + scheduleLog.clean(); + } + + private void clean() { + if (!config.isDeleteExpiredLogsEnable()) return; + try { + cleanMessageLog(); + cleanDispatchLog(scheduleLog::clean); + cleanScheduleOldLog(); + } catch (Throwable e) { + LOGGER.error("LogCleaner exec clean error.", e); + } + } + + @Override + public void start() { + cleanScheduler.scheduleAtFixedRate(this::clean, 0, config.getLogCleanerIntervalSeconds(), TimeUnit.SECONDS); + } + + @Override + public void shutdown() { + cleanScheduler.shutdown(); + try { + cleanScheduler.awaitTermination(10, TimeUnit.SECONDS); + } catch (InterruptedException e) { + LOGGER.error("Shutdown log cleaner scheduler interrupted."); + } + } + + public interface CleanHook { + boolean clean(long key); + } + +} diff --git a/qmq-delay-server/src/main/java/qunar/tc/qmq/delay/config/DefaultStoreConfiguration.java b/qmq-delay-server/src/main/java/qunar/tc/qmq/delay/config/DefaultStoreConfiguration.java new file mode 100644 index 00000000..2778647f --- /dev/null +++ b/qmq-delay-server/src/main/java/qunar/tc/qmq/delay/config/DefaultStoreConfiguration.java @@ -0,0 +1,145 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.delay.config; + +import qunar.tc.qmq.configuration.DynamicConfig; +import qunar.tc.qmq.configuration.DynamicConfigLoader; +import qunar.tc.qmq.constants.BrokerConstants; + +import java.io.File; +import java.util.concurrent.TimeUnit; + +/** + * @author xufeng.deng dennisdxf@gmail.com + * @since 2018-07-11 9:54 + */ +public class DefaultStoreConfiguration implements StoreConfiguration { + private static final String MESSAGE_LOG = "message_log"; + private static final String SCHEDULE_LOG = "schedule_log"; + private static final String DISPATCH_LOG = "dispatch_log"; + private static final String CHECKPOINT = "checkpoint"; + private static final String PER_MESSAGE_SEGMENT_FILE_SIZE = "per.segment.file.size"; + private static final String PER_MESSAGE_SILE = "per.message.size"; + private static final String LOAD_SEGMENT_DELAY_MIN = "load.segment.delay.min"; + private static final String DISPATCH_LOG_KEEP_HOUR = "dispatch.log.keep.hour"; + private static final String SCHEDULE_CLEAN_BEFORE_DISPATCH_HOUR = "schedule.clean.before.dispatch.hour"; + private static final String LOAD_IN_ADVANCE_MIN = "load.in.advance.min"; + private static final String LOAD_BLOCKING_EXIT_MIN = "load.blocking.exit.min"; + + private static final int MESSAGE_SEGMENT_LOG_FILE_SIZE = 1024 * 1024 * 1024; + private static final int SINGLE_MESSAGE_LIMIT_SIZE = 50 * 1024 * 1024; + private static final int SEGMENT_LOAD_DELAY_TIMES_IN_MIN = 1; + private static final int DISPATCH_LOG_KEEP_TIMES_IN_HOUR = 3 * 24; + private static final int SCHEDULE_CLEAN_BEFORE_DISPATCH_TIMES_IN_HOUR = 24; + private static final int LOAD_IN_ADVANCE_TIMES_IN_MIN = 30; + private static final int LOAD_BLOCKING_EXIT_TIMES_IN_MIN = 10; + + private static final long MS_PER_HOUR = TimeUnit.HOURS.toMillis(1); + private static final long MS_PER_MINUTE = TimeUnit.MINUTES.toMillis(1); + + private final DynamicConfig config; + + public DefaultStoreConfiguration(DynamicConfig config) { + this.config = config; + } + + @Override + public DynamicConfig getConfig() { + return config; + } + + @Override + public String getMessageLogStorePath() { + return buildStorePath(MESSAGE_LOG); + } + + @Override + public String getScheduleLogStorePath() { + return buildStorePath(SCHEDULE_LOG); + } + + @Override + public String getDispatchLogStorePath() { + return buildStorePath(DISPATCH_LOG); + } + + @Override + public int getMessageLogSegmentFileSize() { + return config.getInt(PER_MESSAGE_SEGMENT_FILE_SIZE, MESSAGE_SEGMENT_LOG_FILE_SIZE); + } + + @Override + public int getSingleMessageLimitSize() { + return config.getInt(PER_MESSAGE_SILE, SINGLE_MESSAGE_LIMIT_SIZE); + } + + @Override + public String getCheckpointStorePath() { + return buildStorePath(CHECKPOINT); + } + + @Override + public long getMessageLogRetentionMs() { + final int retentionHours = config.getInt(BrokerConstants.MESSAGE_LOG_RETENTION_HOURS, BrokerConstants.DEFAULT_MESSAGE_LOG_RETENTION_HOURS); + return retentionHours * MS_PER_HOUR; + } + + @Override + public long getDispatchLogKeepTime() { + return config.getInt(DISPATCH_LOG_KEEP_HOUR, DISPATCH_LOG_KEEP_TIMES_IN_HOUR) * MS_PER_HOUR; + } + + @Override + public long getCheckCleanTimeBeforeDispatch() { + return config.getInt(SCHEDULE_CLEAN_BEFORE_DISPATCH_HOUR, SCHEDULE_CLEAN_BEFORE_DISPATCH_TIMES_IN_HOUR) * MS_PER_HOUR; + } + + @Override + public long getLogCleanerIntervalSeconds() { + return config.getInt(BrokerConstants.LOG_RETENTION_CHECK_INTERVAL_SECONDS, BrokerConstants.DEFAULT_LOG_RETENTION_CHECK_INTERVAL_SECONDS); + } + + @Override + public String getScheduleOffsetCheckpointPath() { + return buildStorePath(CHECKPOINT); + } + + @Override + public long getLoadInAdvanceTimesInMillis() { + return config.getInt(LOAD_IN_ADVANCE_MIN, LOAD_IN_ADVANCE_TIMES_IN_MIN) * MS_PER_MINUTE; + } + + @Override + public long getLoadBlockingExitTimesInMillis() { + return config.getInt(LOAD_BLOCKING_EXIT_MIN, LOAD_BLOCKING_EXIT_TIMES_IN_MIN) * MS_PER_MINUTE; + } + + @Override + public boolean isDeleteExpiredLogsEnable() { + return config.getBoolean(BrokerConstants.ENABLE_DELETE_EXPIRED_LOGS, false); + } + + @Override + public int getLoadSegmentDelayMinutes() { + return config.getInt(LOAD_SEGMENT_DELAY_MIN, SEGMENT_LOAD_DELAY_TIMES_IN_MIN); + } + + private String buildStorePath(final String name) { + final String root = config.getString(BrokerConstants.STORE_ROOT, BrokerConstants.LOG_STORE_ROOT); + return new File(root, name).getAbsolutePath(); + } +} diff --git a/qmq-delay-server/src/main/java/qunar/tc/qmq/delay/config/StoreConfiguration.java b/qmq-delay-server/src/main/java/qunar/tc/qmq/delay/config/StoreConfiguration.java new file mode 100644 index 00000000..522f219a --- /dev/null +++ b/qmq-delay-server/src/main/java/qunar/tc/qmq/delay/config/StoreConfiguration.java @@ -0,0 +1,57 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.delay.config; + +import qunar.tc.qmq.configuration.DynamicConfig; + +/** + * @author xufeng.deng dennisdxf@gmail.com + * @since 2018-07-11 9:48 + */ +public interface StoreConfiguration { + DynamicConfig getConfig(); + + String getMessageLogStorePath(); + + String getScheduleLogStorePath(); + + String getDispatchLogStorePath(); + + int getMessageLogSegmentFileSize(); + + int getSingleMessageLimitSize(); + + String getCheckpointStorePath(); + + long getMessageLogRetentionMs(); + + int getLoadSegmentDelayMinutes(); + + long getDispatchLogKeepTime(); + + long getCheckCleanTimeBeforeDispatch(); + + long getLogCleanerIntervalSeconds(); + + String getScheduleOffsetCheckpointPath(); + + long getLoadInAdvanceTimesInMillis(); + + long getLoadBlockingExitTimesInMillis(); + + boolean isDeleteExpiredLogsEnable(); +} diff --git a/qmq-delay-server/src/main/java/qunar/tc/qmq/delay/container/Bootstrap.java b/qmq-delay-server/src/main/java/qunar/tc/qmq/delay/container/Bootstrap.java new file mode 100644 index 00000000..b002baca --- /dev/null +++ b/qmq-delay-server/src/main/java/qunar/tc/qmq/delay/container/Bootstrap.java @@ -0,0 +1,26 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.delay.container; + +import qunar.tc.qmq.delay.startup.ServerWrapper; + +public class Bootstrap { + public static void main(String[] args) { + ServerWrapper wrapper = new ServerWrapper(); + wrapper.start(); + } +} diff --git a/qmq-delay-server/src/main/java/qunar/tc/qmq/delay/meta/BrokerRoleManager.java b/qmq-delay-server/src/main/java/qunar/tc/qmq/delay/meta/BrokerRoleManager.java new file mode 100644 index 00000000..e29db402 --- /dev/null +++ b/qmq-delay-server/src/main/java/qunar/tc/qmq/delay/meta/BrokerRoleManager.java @@ -0,0 +1,35 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.delay.meta; + +import qunar.tc.qmq.configuration.BrokerConfig; +import qunar.tc.qmq.meta.BrokerRole; + +/** + * @author xufeng.deng dennisdxf@gmail.com + * @since 2018-10-10 15:05 + */ +public class BrokerRoleManager { + + public static boolean isDelayMaster() { + return BrokerConfig.getBrokerRole() == BrokerRole.DELAY_MASTER; + } + + public static boolean isDelaySlave() { + return BrokerConfig.getBrokerRole() == BrokerRole.DELAY_SLAVE; + } +} diff --git a/qmq-delay-server/src/main/java/qunar/tc/qmq/delay/monitor/QMon.java b/qmq-delay-server/src/main/java/qunar/tc/qmq/delay/monitor/QMon.java new file mode 100644 index 00000000..ee4d519b --- /dev/null +++ b/qmq-delay-server/src/main/java/qunar/tc/qmq/delay/monitor/QMon.java @@ -0,0 +1,124 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.delay.monitor; + +import qunar.tc.qmq.metrics.Metrics; + +import java.util.concurrent.TimeUnit; + +/** + * 需要统一一下,TODO server中的QMon抽离到公共模块 + * + * @author xufeng.deng dennisdxf@gmail.com + * @since 2018-07-26 14:49 + */ +public class QMon { + private static final String[] EMPTY = new String[]{}; + + private static final String[] SUBJECT_ARRAY = new String[]{"subject"}; + private static final String[] BROKER_ARRAY = new String[]{"broker"}; + private static final String[] LOGTYPE_ARRAY = new String[]{"logType"}; + private static final String[] BROKER_GROUP_SUBJECT_ARRAY = new String[]{"group", "subject"}; + + public static void scheduleDispatch() { + Metrics.counter("scheduleDispatch", EMPTY, EMPTY).inc(); + } + + public static void rejectReceivedMessageCountInc(String subject) { + countInc("rejectReceivedMessageCount", subject); + } + + public static void delayBrokerReadOnlyMessageCountInc(String subject) { + countInc("delayBrokerReadOnlyMessageCount", subject); + } + + public static void nettySendMessageFailCount(String subject, String groupName) { + countInc("nettySendMessageFailCount", BROKER_GROUP_SUBJECT_ARRAY, new String[]{groupName, subject}); + } + + public static void appendFailed(String subject) { + countInc("appendMessageLogFailCount", subject); + } + + public static void receivedMessagesCountInc(String subject) { + final String[] values = {subject}; + countInc("receivedMessagesCount", SUBJECT_ARRAY, values); + Metrics.meter("receivedMessagesEx", SUBJECT_ARRAY, values).mark(); + } + + public static void produceTime(String subject, long time) { + Metrics.timer("produceTime", SUBJECT_ARRAY, new String[]{subject}).update(time, TimeUnit.MILLISECONDS); + } + + public static void delayTime(String group, String subject, long time) { + Metrics.timer("delayTime", BROKER_GROUP_SUBJECT_ARRAY, new String[]{group, subject}).update(time, TimeUnit.MILLISECONDS); + } + + public static void receivedRetryMessagesCountInc(String subject) { + final String[] values = {subject}; + countInc("receivedRetryMessagesCount", SUBJECT_ARRAY, values); + } + + public static void delayBrokerSendMsgCount(String groupName, String subject) { + countInc("delayBrokerSendMsgCount", BROKER_GROUP_SUBJECT_ARRAY, new String[]{groupName, subject}); + } + + private static void countInc(String name, String subject) { + countInc(name, SUBJECT_ARRAY, new String[]{subject}); + } + + public static void appendTimer(String subject, long time) { + Metrics.timer("appendMessageLogCostTime", SUBJECT_ARRAY, new String[]{subject}).update(time, TimeUnit.MILLISECONDS); + } + + private static void countInc(String name, String[] tags, String[] values) { + Metrics.counter(name, tags, values).inc(); + } + + public static void sendMsgTime(String broker, long time) { + Metrics.timer("sendMsgTime", BROKER_ARRAY, new String[]{broker}).update(time, TimeUnit.MILLISECONDS); + } + + public static void receiveFailedCuntInc(String subject) { + countInc("receivedFailedCount", subject); + } + + public static void expiredMessagesCountInc(String subject) { + countInc("expiredMessagesCount", subject); + } + + public static void overDelay(String subject) { + countInc("overDelayMessage", subject); + } + + public static void slaveSyncLogOffset(String logType, long diff) { + Metrics.timer("slaveSyncLogOffsetLag", LOGTYPE_ARRAY, new String[]{logType}).update(diff, TimeUnit.MILLISECONDS); + } + + public static void receivedIllegalSubjectMessagesCountInc(String subject) { + final String[] values = {subject}; + countInc("receivedIllegalSubjectMessagesCount", SUBJECT_ARRAY, values); + } + + public static void loadSegmentFailed() { + countInc("loadScheduleSegmentFailed", EMPTY, EMPTY); + } + + public static void appendFailedByMessageIllegal(String subject) { + countInc("appendMessageFailedByIllegal", subject); + } +} diff --git a/qmq-delay-server/src/main/java/qunar/tc/qmq/delay/receiver/Invoker.java b/qmq-delay-server/src/main/java/qunar/tc/qmq/delay/receiver/Invoker.java new file mode 100644 index 00000000..467536f5 --- /dev/null +++ b/qmq-delay-server/src/main/java/qunar/tc/qmq/delay/receiver/Invoker.java @@ -0,0 +1,27 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.delay.receiver; + +import qunar.tc.qmq.delay.base.ReceivedDelayMessage; + +/** + * @author xufeng.deng dennisdxf@gmail.com + * @since 2018-07-26 14:28 + */ +public interface Invoker { + void invoke(ReceivedDelayMessage message); +} diff --git a/qmq-delay-server/src/main/java/qunar/tc/qmq/delay/receiver/ReceivedDelayMessageProcessor.java b/qmq-delay-server/src/main/java/qunar/tc/qmq/delay/receiver/ReceivedDelayMessageProcessor.java new file mode 100644 index 00000000..94a37238 --- /dev/null +++ b/qmq-delay-server/src/main/java/qunar/tc/qmq/delay/receiver/ReceivedDelayMessageProcessor.java @@ -0,0 +1,212 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.delay.receiver; + +import com.google.common.collect.Lists; +import com.google.common.util.concurrent.FutureCallback; +import com.google.common.util.concurrent.Futures; +import com.google.common.util.concurrent.ListenableFuture; +import com.google.common.util.concurrent.MoreExecutors; +import io.netty.buffer.ByteBuf; +import io.netty.buffer.Unpooled; +import io.netty.channel.ChannelHandlerContext; +import qunar.tc.qmq.base.BaseMessage; +import qunar.tc.qmq.base.MessageHeader; +import qunar.tc.qmq.delay.meta.BrokerRoleManager; +import qunar.tc.qmq.delay.store.model.RawMessageExtend; +import qunar.tc.qmq.netty.NettyRequestProcessor; +import qunar.tc.qmq.protocol.Datagram; +import qunar.tc.qmq.protocol.RemotingCommand; +import qunar.tc.qmq.protocol.RemotingHeader; +import qunar.tc.qmq.utils.CharsetUtils; +import qunar.tc.qmq.utils.Crc32; +import qunar.tc.qmq.utils.Flags; + +import java.util.Collections; +import java.util.List; +import java.util.concurrent.CompletableFuture; + +import static qunar.tc.qmq.protocol.QMQSerializer.deserializeMessageHeader; + +/** + * @author xufeng.deng dennisdxf@gmail.com + * @since 2018-07-27 17:36 + */ +public class ReceivedDelayMessageProcessor implements NettyRequestProcessor { + private final Receiver receiver; + + public ReceivedDelayMessageProcessor(Receiver receiver) { + this.receiver = receiver; + } + + public static List deserializeRawMessagesExtend(RemotingCommand request) { + final ByteBuf body = request.getBody(); + if (body.readableBytes() == 0) return Collections.emptyList(); + + List messages = Lists.newArrayList(); + short version = request.getHeader().getVersion(); + while (body.isReadable()) { + if (version >= RemotingHeader.VERSION_7) { + messages.add(doDeserializeRawMessagesExtend(body)); + } else if (version == RemotingHeader.VERSION_6) { + messages.add(deserializeRawMessageExtendWithCrc(body)); + } else if (version == RemotingHeader.VERSION_5) { + messages.add(deserializeRawMessageExtendIgnoreCrc(body)); + } else { + messages.add(deserializeRawMessageExtend(body)); + } + } + return messages; + } + + private static RawMessageExtend doDeserializeRawMessagesExtend(ByteBuf body) { + body.markReaderIndex(); + int headerStart = body.readerIndex(); + long bodyCrc = body.readLong(); + MessageHeader header = deserializeMessageHeader(body); + header.setBodyCrc(bodyCrc); + int bodyLen = body.readInt(); + int headerLen = body.readerIndex() - headerStart; + int totalLen = headerLen + bodyLen; + + body.resetReaderIndex(); + ByteBuf messageBuf = body.readSlice(totalLen); + // client config error,prefer to send after ten second + long scheduleTime = System.currentTimeMillis() + 10000; + if (Flags.isDelay(header.getFlag())) { + scheduleTime = header.getExpireTime(); + } + + return new RawMessageExtend(header, messageBuf, messageBuf.readableBytes(), scheduleTime); + } + + private static RawMessageExtend deserializeRawMessageExtendWithCrc(ByteBuf body) { + body.markReaderIndex(); + int headerStart = body.readerIndex(); + long bodyCrc = body.readLong(); + MessageHeader header = deserializeMessageHeader(body); + header.setBodyCrc(bodyCrc); + int bodyLen = body.readInt(); + int headerLen = body.readerIndex() - headerStart; + int totalLen = headerLen + bodyLen; + long scheduleTime = getScheduleTime(bodyLen, body); + + body.resetReaderIndex(); + ByteBuf messageBuf = body.readSlice(totalLen); + return new RawMessageExtend(header, messageBuf, messageBuf.readableBytes(), scheduleTime); + } + + private static long getScheduleTime(int bodyLen, ByteBuf body) { + ByteBuf buffer = body.readSlice(bodyLen); + while (buffer.isReadable(Short.BYTES)) { + short keySize = buffer.readShort(); + if (!buffer.isReadable(keySize)) return -1; + byte[] keyBs = new byte[keySize]; + buffer.readBytes(keyBs); + + if (!buffer.isReadable(Short.BYTES)) return -1; + int valSize = buffer.readShort(); + if (!buffer.isReadable(valSize)) return -1; + + String key = CharsetUtils.toUTF8String(keyBs); + if (BaseMessage.keys.qmq_scheduleReceiveTime.name().equalsIgnoreCase(key)) { + byte[] valBs = new byte[valSize]; + buffer.readBytes(valBs); + return Long.valueOf(CharsetUtils.toUTF8String(valBs)); + } else { + buffer.skipBytes(valSize); + } + } + + return -1; + } + + private static RawMessageExtend deserializeRawMessageExtendIgnoreCrc(ByteBuf body) { + //skip wrong crc + body.readLong(); + + body.markReaderIndex(); + int headerStart = body.readerIndex(); + + MessageHeader header = deserializeMessageHeader(body); + int bodyLen = body.readInt(); + int headerLen = body.readerIndex() - headerStart; + int totalLen = headerLen + bodyLen; + + long scheduleTime = getScheduleTime(bodyLen, body); + + body.resetReaderIndex(); + byte[] data = new byte[totalLen]; + body.readBytes(data); + long crc = Crc32.crc32(data); + header.setBodyCrc(crc); + + ByteBuf buffer = Unpooled.buffer(8 + totalLen); + buffer.writeLong(crc); + buffer.writeBytes(data); + + return new RawMessageExtend(header, buffer, buffer.readableBytes(), scheduleTime); + } + + private static RawMessageExtend deserializeRawMessageExtend(ByteBuf body) { + body.markReaderIndex(); + int headerStart = body.readerIndex(); + MessageHeader header = deserializeMessageHeader(body); + + int bodyLen = body.readInt(); + int headerLen = body.readerIndex() - headerStart; + int totalLen = headerLen + bodyLen; + + long scheduleTime = getScheduleTime(bodyLen, body); + + body.resetReaderIndex(); + byte[] data = new byte[totalLen]; + body.readBytes(data); + long crc = Crc32.crc32(data); + header.setBodyCrc(crc); + + ByteBuf buffer = Unpooled.buffer(8 + totalLen); + buffer.writeLong(crc); + buffer.writeBytes(data); + + return new RawMessageExtend(header, buffer, buffer.readableBytes(), scheduleTime); + } + + @Override + public CompletableFuture processRequest(ChannelHandlerContext ctx, RemotingCommand request) { + final List messages = deserializeRawMessagesExtend(request); + final ListenableFuture result = receiver.receive(messages, request); + final CompletableFuture future = new CompletableFuture<>(); + Futures.addCallback(result, new FutureCallback() { + @Override + public void onSuccess(Datagram datagram) { + future.complete(datagram); + } + + @Override + public void onFailure(Throwable ex) { + future.completeExceptionally(ex); + } + }, MoreExecutors.directExecutor()); + return future; + } + + @Override + public boolean rejectRequest() { + return !BrokerRoleManager.isDelayMaster(); + } +} diff --git a/qmq-delay-server/src/main/java/qunar/tc/qmq/delay/receiver/Receiver.java b/qmq-delay-server/src/main/java/qunar/tc/qmq/delay/receiver/Receiver.java new file mode 100644 index 00000000..9b890927 --- /dev/null +++ b/qmq-delay-server/src/main/java/qunar/tc/qmq/delay/receiver/Receiver.java @@ -0,0 +1,270 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.delay.receiver; + +import com.google.common.base.CharMatcher; +import com.google.common.eventbus.Subscribe; +import com.google.common.util.concurrent.Futures; +import com.google.common.util.concurrent.ListenableFuture; +import com.google.common.util.concurrent.SettableFuture; +import io.netty.buffer.ByteBuf; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import qunar.tc.qmq.base.MessageHeader; +import qunar.tc.qmq.configuration.BrokerConfig; +import qunar.tc.qmq.configuration.DynamicConfig; +import qunar.tc.qmq.delay.DelayLogFacade; +import qunar.tc.qmq.delay.base.ReceivedDelayMessage; +import qunar.tc.qmq.delay.base.ReceivedResult; +import qunar.tc.qmq.delay.monitor.QMon; +import qunar.tc.qmq.delay.receiver.filter.OverDelayException; +import qunar.tc.qmq.delay.receiver.filter.ReceiveFilterChain; +import qunar.tc.qmq.delay.store.model.RawMessageExtend; +import qunar.tc.qmq.protocol.*; +import qunar.tc.qmq.protocol.producer.MessageProducerCode; +import qunar.tc.qmq.sync.DelaySyncRequest; +import qunar.tc.qmq.util.RemotingBuilder; +import qunar.tc.qmq.utils.CharsetUtils; +import qunar.tc.qmq.utils.RetrySubjectUtils; + +import java.util.ArrayList; +import java.util.Deque; +import java.util.List; +import java.util.Objects; +import java.util.concurrent.LinkedBlockingDeque; + +import static qunar.tc.qmq.delay.receiver.filter.OverDelayFilter.TWO_YEAR_MILLIS; + +/** + * @author xufeng.deng dennisdxf@gmail.com + * @since 2018-07-26 14:46 + */ +public class Receiver { + private static final Logger LOGGER = LoggerFactory.getLogger(Receiver.class); + + private final DelayLogFacade facade; + private final DynamicConfig config; + private final Invoker invoker; + private final Deque waitSlaveSyncQueue = new LinkedBlockingDeque<>(); + + public Receiver(DynamicConfig config, DelayLogFacade facade) { + this.config = config; + this.facade = facade; + this.invoker = new ReceiveFilterChain().buildFilterChain(this::doInvoke); + } + + ListenableFuture receive(List messages, RemotingCommand cmd) { + final List> futures = new ArrayList<>(messages.size()); + for (RawMessageExtend message : messages) { + final MessageHeader header = message.getHeader(); + monitorMessageReceived(header.getCreateTime(), header.getSubject()); + + final ReceivedDelayMessage receivedDelayMessage = new ReceivedDelayMessage(message, cmd.getReceiveTime()); + futures.add(receivedDelayMessage.getPromise()); + try { + invoker.invoke(receivedDelayMessage); + } catch (OverDelayException e) { + overDelay(receivedDelayMessage); + } + } + + final short version = cmd.getHeader().getVersion(); + return Futures.transform(Futures.allAsList(futures) + , results -> RemotingBuilder.buildResponseDatagram(CommandCode.SUCCESS + , cmd.getHeader(), new SendResultPayloadHolder(results, version))); + } + + private void doInvoke(ReceivedDelayMessage message) { + if (BrokerConfig.isReadonly()) { + brokerReadOnly(message); + return; + } + + if (bigSlaveLag()) { + brokerReadOnly(message); + return; + } + + if (isSubjectIllegal(message.getSubject())) { + if (allowRejectIllegalSubject()) { + notAllowed(message); + return; + } + } + + try { + ReceivedResult result = facade.appendMessageLog(message); + offer(message, result); + } catch (Throwable t) { + error(message, t); + } + } + + private void monitorMessageReceived(long receiveTime, String subject) { + if (RetrySubjectUtils.isRetrySubject(subject) || RetrySubjectUtils.isDeadRetrySubject(subject)) { + QMon.receivedRetryMessagesCountInc(subject); + } + QMon.receivedMessagesCountInc(subject); + QMon.produceTime(subject, System.currentTimeMillis() - receiveTime); + } + + @Subscribe + @SuppressWarnings("unused") + public void syncRequest(DelaySyncRequest syncRequest) { + long messageLogOffset = syncRequest.getMessageLogOffset(); + ReceiveEntry first; + while ((first = this.waitSlaveSyncQueue.peekFirst()) != null) { + if (first.result.getMessageLogOffset() > messageLogOffset) { + break; + } + + this.waitSlaveSyncQueue.pop(); + end(first.message, first.result); + } + } + + private void notAllowed(ReceivedDelayMessage message) { + QMon.rejectReceivedMessageCountInc(message.getSubject()); + end(message, new ReceivedResult(message.getMessageId(), MessageProducerCode.SUBJECT_NOT_ASSIGNED, "message rejected")); + } + + private boolean isSubjectIllegal(final String subject) { + final String trimmedSubject = CharMatcher.INVISIBLE.trimFrom(subject); + if (Objects.equals(subject, trimmedSubject)) { + return false; + } + + QMon.receivedIllegalSubjectMessagesCountInc(trimmedSubject); + LOGGER.error("received message with illegal subject. subject: {}", subject); + return true; + } + + private boolean allowRejectIllegalSubject() { + return config.getBoolean("Receiver.RejectIllegalSubject", false); + } + + private boolean bigSlaveLag() { + return shouldWaitSlave() && waitSlaveSyncQueue.size() >= config.getInt("receive.queue.size", 50000); + } + + private boolean shouldWaitSlave() { + return config.getBoolean("wait.slave.wrote", false); + } + + private void brokerReadOnly(ReceivedDelayMessage message) { + QMon.delayBrokerReadOnlyMessageCountInc(message.getSubject()); + end(message, new ReceivedResult(message.getMessageId(), MessageProducerCode.BROKER_READ_ONLY, "BROKER_READ_ONLY")); + } + + private void offer(ReceivedDelayMessage message, ReceivedResult result) { + if (MessageProducerCode.SUCCESS != result.getCode()) { + end(message, result); + return; + } + + if (!shouldWaitSlave()) { + end(message, result); + return; + } + + waitSlaveSyncQueue.addLast(new ReceiveEntry(message, result)); + } + + private void overDelay(final ReceivedDelayMessage message) { + LOGGER.warn("received delay message over delay,message:{}", message); + QMon.overDelay(message.getSubject()); + adjustScheduleTime(message); + } + + private void adjustScheduleTime(final ReceivedDelayMessage message) { + long now = System.currentTimeMillis(); + message.adjustScheduleTime(now + TWO_YEAR_MILLIS); + } + + private void error(ReceivedDelayMessage message, Throwable e) { + LOGGER.error("delay broker receive message error,subject:{} ,id:{} ,msg:{}", message.getSubject(), message.getMessageId(), message, e); + QMon.receiveFailedCuntInc(message.getSubject()); + end(message, new ReceivedResult(message.getMessageId(), MessageProducerCode.STORE_ERROR, "store error")); + } + + private void end(ReceivedDelayMessage message, ReceivedResult result) { + try { + message.done(result); + } catch (Throwable e) { + LOGGER.error("send response failed id:{} ,msg:{}", message.getMessageId(), message); + } + } + + public static class SendResultPayloadHolder implements PayloadHolder { + private final List results; + + private final short version; + + public SendResultPayloadHolder(List results, short version) { + this.results = results; + this.version = version; + } + + @Override + public void writeBody(ByteBuf out) { + // VERSION_4 以下协议,返回所有消息信息 + if (version < RemotingHeader.VERSION_4) { + writeBodyV3(out); + } else { + for (ReceivedResult result : results) { + if (MessageProducerCode.SUCCESS != result.getCode()) { + writeItem(result, out); + } + } + } + } + + private void writeBodyV3(ByteBuf out) { + for (ReceivedResult result : results) { + writeItem(result, out); + } + } + + private void writeItem(ReceivedResult result, ByteBuf out) { + int code = result.getCode(); + writeString(result.getMessageId(), out); + out.writeInt(code); + writeString(result.getRemark(), out); + } + + private void writeString(String str, ByteBuf out) { + byte[] bytes = CharsetUtils.toUTF8Bytes(str); + if (bytes != null) { + out.writeShort((short) bytes.length); + out.writeBytes(bytes); + } else { + out.writeShort(0); + } + } + } + + private static class ReceiveEntry { + final ReceivedDelayMessage message; + final ReceivedResult result; + + ReceiveEntry(ReceivedDelayMessage message, ReceivedResult result) { + this.message = message; + this.result = result; + } + } + +} diff --git a/qmq-delay-server/src/main/java/qunar/tc/qmq/delay/receiver/filter/Filter.java b/qmq-delay-server/src/main/java/qunar/tc/qmq/delay/receiver/filter/Filter.java new file mode 100644 index 00000000..1d489ca3 --- /dev/null +++ b/qmq-delay-server/src/main/java/qunar/tc/qmq/delay/receiver/filter/Filter.java @@ -0,0 +1,28 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.delay.receiver.filter; + +import qunar.tc.qmq.delay.base.ReceivedDelayMessage; +import qunar.tc.qmq.delay.receiver.Invoker; + +/** + * @author xufeng.deng dennisdxf@gmail.com + * @since 2018-07-26 14:28 + */ +public interface Filter { + void invoke(Invoker invoker, ReceivedDelayMessage message); +} diff --git a/qmq-delay-server/src/main/java/qunar/tc/qmq/delay/receiver/filter/OverDelayException.java b/qmq-delay-server/src/main/java/qunar/tc/qmq/delay/receiver/filter/OverDelayException.java new file mode 100644 index 00000000..43f8650f --- /dev/null +++ b/qmq-delay-server/src/main/java/qunar/tc/qmq/delay/receiver/filter/OverDelayException.java @@ -0,0 +1,28 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.delay.receiver.filter; + +/** + * @author xufeng.deng dennisdxf@gmail.com + * @since 2018-08-01 17:10 + */ +public class OverDelayException extends RuntimeException { + + OverDelayException(String message) { + super(message); + } +} diff --git a/qmq-delay-server/src/main/java/qunar/tc/qmq/delay/receiver/filter/OverDelayFilter.java b/qmq-delay-server/src/main/java/qunar/tc/qmq/delay/receiver/filter/OverDelayFilter.java new file mode 100644 index 00000000..b799198c --- /dev/null +++ b/qmq-delay-server/src/main/java/qunar/tc/qmq/delay/receiver/filter/OverDelayFilter.java @@ -0,0 +1,41 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.delay.receiver.filter; + +import qunar.tc.qmq.delay.base.ReceivedDelayMessage; +import qunar.tc.qmq.delay.receiver.Invoker; + +import java.util.concurrent.TimeUnit; + + +/** + * @author xufeng.deng dennisdxf@gmail.com + * @since 2018-08-01 16:26 + */ +public class OverDelayFilter implements Filter { + public static final long TWO_YEAR_MILLIS = TimeUnit.DAYS.toMillis(365 * 2); + + @Override + public void invoke(Invoker invoker, ReceivedDelayMessage message) { + if (message.getScheduleTime() > (System.currentTimeMillis() + TWO_YEAR_MILLIS)) { + throw new OverDelayException("messageOverDelay"); + } + + invoker.invoke(message); + } + +} diff --git a/qmq-delay-server/src/main/java/qunar/tc/qmq/delay/receiver/filter/PastDelayFilter.java b/qmq-delay-server/src/main/java/qunar/tc/qmq/delay/receiver/filter/PastDelayFilter.java new file mode 100644 index 00000000..82af309b --- /dev/null +++ b/qmq-delay-server/src/main/java/qunar/tc/qmq/delay/receiver/filter/PastDelayFilter.java @@ -0,0 +1,36 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.delay.receiver.filter; + +import qunar.tc.qmq.delay.base.ReceivedDelayMessage; +import qunar.tc.qmq.delay.receiver.Invoker; + +/** + * @author xufeng.deng dennisdxf@gmail.com + * @since 2018-09-06 11:36 + */ +public class PastDelayFilter implements Filter { + @Override + public void invoke(Invoker invoker, ReceivedDelayMessage message) { + long now = System.currentTimeMillis(); + if (message.getScheduleTime() < now) { + message.adjustScheduleTime(now); + } + + invoker.invoke(message); + } +} diff --git a/qmq-delay-server/src/main/java/qunar/tc/qmq/delay/receiver/filter/ReceiveFilterChain.java b/qmq-delay-server/src/main/java/qunar/tc/qmq/delay/receiver/filter/ReceiveFilterChain.java new file mode 100644 index 00000000..c118b009 --- /dev/null +++ b/qmq-delay-server/src/main/java/qunar/tc/qmq/delay/receiver/filter/ReceiveFilterChain.java @@ -0,0 +1,54 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.delay.receiver.filter; + +import com.google.common.collect.Lists; +import qunar.tc.qmq.delay.receiver.Invoker; + +import java.util.List; + +/** + * 接收过滤器链 + * + * @author kelly.li + */ +public class ReceiveFilterChain { + private List filters = Lists.newArrayList(); + + public ReceiveFilterChain() { + addFilter(new ValidateFilter()); + addFilter(new OverDelayFilter()); + addFilter(new PastDelayFilter()); + } + + public Invoker buildFilterChain(Invoker invoker) { + Invoker last = invoker; + if (0 < filters.size()) { + for (int i = filters.size() - 1; i >= 0; i--) { + final Filter filter = filters.get(i); + final Invoker next = last; + last = message -> filter.invoke(next, message); + } + } + + return last; + } + + private void addFilter(Filter filter) { + filters.add(filter); + } +} diff --git a/qmq-delay-server/src/main/java/qunar/tc/qmq/delay/receiver/filter/ValidateFilter.java b/qmq-delay-server/src/main/java/qunar/tc/qmq/delay/receiver/filter/ValidateFilter.java new file mode 100644 index 00000000..64613f6e --- /dev/null +++ b/qmq-delay-server/src/main/java/qunar/tc/qmq/delay/receiver/filter/ValidateFilter.java @@ -0,0 +1,38 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.delay.receiver.filter; + +import com.google.common.base.Preconditions; +import com.google.common.base.Strings; +import qunar.tc.qmq.delay.base.ReceivedDelayMessage; +import qunar.tc.qmq.delay.receiver.Invoker; + +/** + * 验证过滤器 + * + * @author kelly.li + */ +public class ValidateFilter implements Filter { + @Override + public void invoke(Invoker invoker, ReceivedDelayMessage message) { + Preconditions.checkNotNull(message, "message not null"); + Preconditions.checkArgument(!Strings.isNullOrEmpty(message.getMessageId()), "message id should not be empty"); + Preconditions.checkArgument(!Strings.isNullOrEmpty(message.getSubject()), "message subject should not be empty"); + + invoker.invoke(message); + } +} diff --git a/qmq-delay-server/src/main/java/qunar/tc/qmq/delay/sender/DelayProcessor.java b/qmq-delay-server/src/main/java/qunar/tc/qmq/delay/sender/DelayProcessor.java new file mode 100644 index 00000000..7c89d6ab --- /dev/null +++ b/qmq-delay-server/src/main/java/qunar/tc/qmq/delay/sender/DelayProcessor.java @@ -0,0 +1,30 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.delay.sender; + +import io.netty.buffer.ByteBuf; +import qunar.tc.qmq.common.Disposable; + +/** + * @author xufeng.deng dennisdxf@gmail.com + * @since 2018-08-07 13:24 + */ +public interface DelayProcessor extends Disposable { + void init(); + + void send(ByteBuf record); +} diff --git a/qmq-delay-server/src/main/java/qunar/tc/qmq/delay/sender/NettySender.java b/qmq-delay-server/src/main/java/qunar/tc/qmq/delay/sender/NettySender.java new file mode 100644 index 00000000..8c772d58 --- /dev/null +++ b/qmq-delay-server/src/main/java/qunar/tc/qmq/delay/sender/NettySender.java @@ -0,0 +1,60 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.delay.sender; + +import qunar.tc.qmq.config.NettyClientConfigManager; +import qunar.tc.qmq.delay.store.model.ScheduleSetRecord; +import qunar.tc.qmq.netty.client.NettyClient; +import qunar.tc.qmq.netty.exception.ClientSendException; +import qunar.tc.qmq.netty.exception.RemoteTimeoutException; +import qunar.tc.qmq.protocol.CommandCode; +import qunar.tc.qmq.protocol.Datagram; +import qunar.tc.qmq.protocol.RemotingHeader; +import qunar.tc.qmq.util.RemotingBuilder; + +import java.util.List; + +/** + * @author xufeng.deng dennisdxf@gmail.com + * @since 2018-07-23 16:33 + */ +public class NettySender implements Sender { + private final NettyClient client; + + public NettySender() { + this.client = NettyClient.getClient(); + this.client.start(NettyClientConfigManager.get().getDefaultClientConfig()); + } + + @Override + public Datagram send(List records, SenderGroup senderGroup) throws InterruptedException, RemoteTimeoutException, ClientSendException { + Datagram requestDatagram = RemotingBuilder.buildRequestDatagram(CommandCode.SEND_MESSAGE, out -> { + if (null == records || records.isEmpty()) { + return; + } + for (ScheduleSetRecord record : records) { + out.writeBytes(record.getRecord()); + } + }); + requestDatagram.getHeader().setVersion(RemotingHeader.VERSION_7); + return client.sendSync(senderGroup.getBrokerGroupInfo().getMaster(), requestDatagram, 5 * 1000); + } + + @Override + public void shutdown() { + } +} diff --git a/qmq-delay-server/src/main/java/qunar/tc/qmq/delay/sender/Sender.java b/qmq-delay-server/src/main/java/qunar/tc/qmq/delay/sender/Sender.java new file mode 100644 index 00000000..a51ab462 --- /dev/null +++ b/qmq-delay-server/src/main/java/qunar/tc/qmq/delay/sender/Sender.java @@ -0,0 +1,34 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.delay.sender; + +import qunar.tc.qmq.delay.store.model.ScheduleSetRecord; +import qunar.tc.qmq.netty.exception.ClientSendException; +import qunar.tc.qmq.netty.exception.RemoteTimeoutException; +import qunar.tc.qmq.protocol.Datagram; + +import java.util.List; + +/** + * @author xufeng.deng dennisdxf@gmail.com + * @since 2018-07-23 11:19 + */ +public interface Sender { + Datagram send(List records, SenderGroup group) throws InterruptedException, RemoteTimeoutException, ClientSendException; + + void shutdown(); +} diff --git a/qmq-delay-server/src/main/java/qunar/tc/qmq/delay/sender/SenderExecutor.java b/qmq-delay-server/src/main/java/qunar/tc/qmq/delay/sender/SenderExecutor.java new file mode 100644 index 00000000..74dfafb4 --- /dev/null +++ b/qmq-delay-server/src/main/java/qunar/tc/qmq/delay/sender/SenderExecutor.java @@ -0,0 +1,120 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.delay.sender; + +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; +import qunar.tc.qmq.broker.BrokerClusterInfo; +import qunar.tc.qmq.broker.BrokerGroupInfo; +import qunar.tc.qmq.broker.BrokerLoadBalance; +import qunar.tc.qmq.broker.BrokerService; +import qunar.tc.qmq.broker.impl.PollBrokerLoadBalance; +import qunar.tc.qmq.common.ClientType; +import qunar.tc.qmq.common.Disposable; +import qunar.tc.qmq.configuration.DynamicConfig; +import qunar.tc.qmq.delay.store.model.ScheduleSetRecord; + +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; + +/** + * @author xufeng.deng dennisdxf@gmail.com + * @since 2018-08-16 21:00 + */ +class SenderExecutor implements Disposable { + private static final int DEFAULT_SEND_THREAD = 1; + + private final ConcurrentMap groupSenders = new ConcurrentHashMap<>(); + private final BrokerLoadBalance brokerLoadBalance; + private final Sender sender; + private final int sendThreads; + + SenderExecutor(final Sender sender, DynamicConfig sendConfig) { + this.sender = sender; + this.brokerLoadBalance = PollBrokerLoadBalance.getInstance(); + this.sendThreads = sendConfig.getInt("delay.send.threads", DEFAULT_SEND_THREAD); + } + + void execute(final List logRecords, final SenderGroup.ResultHandler handler, final BrokerService brokerService) { + Map> records = groupByBroker(logRecords, brokerService); + for (Map.Entry> entry : records.entrySet()) { + doExecute(entry.getKey(), entry.getValue(), handler); + } + } + + private void doExecute(final SenderGroup group, final List records, final SenderGroup.ResultHandler handler) { + group.send(records, sender, handler); + } + + private Map> groupByBroker(final List records, final BrokerService brokerService) { + Map> groups = Maps.newHashMap(); + Map> recordsGroupBySubject = groupBySubject(records); + for (Map.Entry> entry : recordsGroupBySubject.entrySet()) { + List setRecordsGroupBySubject = entry.getValue(); + BrokerGroupInfo groupInfo = loadGroup(entry.getKey(), brokerService); + SenderGroup senderGroup = getGroup(groupInfo, sendThreads); + + List recordsInGroup = groups.get(senderGroup); + if (null == recordsInGroup) { + recordsInGroup = Lists.newArrayListWithCapacity(setRecordsGroupBySubject.size()); + } + recordsInGroup.addAll(setRecordsGroupBySubject); + groups.put(senderGroup, recordsInGroup); + } + + return groups; + } + + private SenderGroup getGroup(BrokerGroupInfo groupInfo, int sendThreads) { + String groupName = groupInfo.getGroupName(); + SenderGroup senderGroup = groupSenders.get(groupName); + if (null == senderGroup) { + senderGroup = new SenderGroup(groupInfo, sendThreads); + SenderGroup currentSenderGroup = groupSenders.putIfAbsent(groupName, senderGroup); + senderGroup = null != currentSenderGroup ? currentSenderGroup : senderGroup; + } else { + senderGroup.reconfigureGroup(groupInfo); + } + + return senderGroup; + } + + private BrokerGroupInfo loadGroup(String subject, BrokerService brokerService) { + BrokerClusterInfo cluster = brokerService.getClusterBySubject(ClientType.PRODUCER, subject); + return brokerLoadBalance.loadBalance(cluster, null); + } + + private Map> groupBySubject(List records) { + Map> map = Maps.newHashMap(); + for (ScheduleSetRecord record : records) { + if (null != record) { + List recordList = map.computeIfAbsent(record.getSubject(), k -> Lists.newArrayList()); + recordList.add(record); + } + } + + return map; + } + + @Override + public void destroy() { + groupSenders.values().parallelStream().forEach(SenderGroup::destroy); + groupSenders.clear(); + } +} diff --git a/qmq-delay-server/src/main/java/qunar/tc/qmq/delay/sender/SenderGroup.java b/qmq-delay-server/src/main/java/qunar/tc/qmq/delay/sender/SenderGroup.java new file mode 100644 index 00000000..dc89e4f3 --- /dev/null +++ b/qmq-delay-server/src/main/java/qunar/tc/qmq/delay/sender/SenderGroup.java @@ -0,0 +1,240 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.delay.sender; + +import com.google.common.collect.Lists; +import com.google.common.util.concurrent.RateLimiter; +import com.google.common.util.concurrent.ThreadFactoryBuilder; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import qunar.tc.qmq.broker.BrokerGroupInfo; +import qunar.tc.qmq.common.Disposable; +import qunar.tc.qmq.delay.base.GroupSendException; +import qunar.tc.qmq.delay.monitor.QMon; +import qunar.tc.qmq.delay.store.model.ScheduleSetRecord; +import qunar.tc.qmq.metrics.Metrics; +import qunar.tc.qmq.netty.exception.ClientSendException; +import qunar.tc.qmq.protocol.CommandCode; +import qunar.tc.qmq.protocol.Datagram; +import qunar.tc.qmq.protocol.QMQSerializer; +import qunar.tc.qmq.protocol.producer.MessageProducerCode; +import qunar.tc.qmq.protocol.producer.SendResult; + +import java.util.*; +import java.util.concurrent.*; +import java.util.concurrent.atomic.AtomicReference; + +import static qunar.tc.qmq.delay.monitor.QMon.delayBrokerSendMsgCount; +import static qunar.tc.qmq.delay.monitor.QMon.delayTime; + +/** + * @author xufeng.deng dennisdxf@gmail.com + * @since 2018-08-16 21:01 + */ +public class SenderGroup implements Disposable { + private static final Logger LOGGER = LoggerFactory.getLogger(SenderGroup.class); + private static final int MAX_SEND_BATCH_SIZE = 50; + private final AtomicReference groupInfo; + private final ThreadPoolExecutor executorService; + private final RateLimiter LOG_LIMITER = RateLimiter.create(2); + + SenderGroup(final BrokerGroupInfo groupInfo, int sendThreads) { + this.groupInfo = new AtomicReference<>(groupInfo); + this.executorService = new ThreadPoolExecutor(1, sendThreads, 1L, TimeUnit.MINUTES, + new LinkedBlockingQueue<>(), new ThreadFactoryBuilder() + .setNameFormat("delay-sender-" + groupInfo.getGroupName() + "-%d").build()); + executorService.setRejectedExecutionHandler(new SenderRejectExecutionHandler(this)); + } + + public void send(final List records, final Sender sender, final ResultHandler handler) { + executorService.execute(() -> doSend(records, sender, handler)); + } + + private void doSend(final List records, final Sender sender, final ResultHandler handler) { + BrokerGroupInfo groupInfo = this.groupInfo.get(); + String groupName = groupInfo.getGroupName(); + List> partitions = Lists.partition(records, MAX_SEND_BATCH_SIZE); + + for (List recordList : partitions) { + try { + Datagram response = sendMessages(recordList, sender); + monitor(recordList, groupName); + if (null == response) { + handler.fail(recordList); + } else { + final int responseCode = response.getHeader().getCode(); + final Map resultMap = getSendResult(response); + + if (null == resultMap || CommandCode.SUCCESS != responseCode) { + if (responseCode == CommandCode.BROKER_REJECT || responseCode == CommandCode.BROKER_ERROR) { + groupInfo.markFailed(); + } + + monitorSendFail(recordList, groupInfo.getGroupName()); + handler.fail(recordList); + return; + } + + Set failedMessageIds = new HashSet<>(); + boolean brokerRefreshed = false; + for (Map.Entry entry1 : resultMap.entrySet()) { + if (entry1.getValue().getCode() != MessageProducerCode.SUCCESS) { + failedMessageIds.add(entry1.getKey()); + } + if (!brokerRefreshed && entry1.getValue().getCode() == MessageProducerCode.BROKER_READ_ONLY) { + groupInfo.markFailed(); + brokerRefreshed = true; + } + } + if (!brokerRefreshed) groupInfo.markSuccess(); + + handler.success(recordList, failedMessageIds); + } + } catch (Throwable e) { + LOGGER.error("sender group send records failed,broker:{},records size:{}", groupName, recordList.size(), e); + throw new GroupSendException(e); + } + } + } + + private void monitor(final List records, final String groupName) { + for (ScheduleSetRecord record : records) { + String subject = record.getSubject(); + long delay = System.currentTimeMillis() - record.getScheduleTime(); + delayBrokerSendMsgCount(groupName, subject); + delayTime(groupName, subject, delay); + } + Metrics.meter("delaySendMessagesQps", new String[]{"group"}, new String[]{groupName}).mark(records.size()); + } + + BrokerGroupInfo getBrokerGroupInfo() { + return groupInfo.get(); + } + + void reconfigureGroup(final BrokerGroupInfo brokerGroup) { + BrokerGroupInfo old = this.groupInfo.get(); + if (!brokerIsEquals(old, brokerGroup)) { + this.groupInfo.set(brokerGroup); + LOGGER.info("netty sender group reconfigure, {} -> {}", old, brokerGroup); + } + } + + private boolean brokerIsEquals(BrokerGroupInfo current, BrokerGroupInfo next) { + return current.getGroupName().equals(next.getGroupName()) + && current.getMaster().equals(next.getMaster()); + } + + private Map getSendResult(Datagram response) { + try { + return QMQSerializer.deserializeSendResultMap(response.getBody()); + } catch (Exception e) { + LOGGER.error("delay broker send exception on deserializeSendResultMap.", e); + return null; + } + } + + private Datagram sendMessages(final List records, final Sender sender) { + long start = System.currentTimeMillis(); + try { + return sender.send(records, this); + } catch (ClientSendException e) { + ClientSendException.SendErrorCode errorCode = e.getSendErrorCode(); + monitorSendError(records, groupInfo.get(), errorCode.ordinal()); + } catch (Exception e) { + monitorSendError(records, groupInfo.get(), -1); + } finally { + QMon.sendMsgTime(groupInfo.get().getGroupName(), System.currentTimeMillis() - start); + } + + return null; + } + + private void monitorSendFail(List records, String groupName) { + records.parallelStream().forEach(record -> monitorSendFail(record.getSubject(), groupName)); + } + + private void monitorSendFail(String subject, String groupName) { + if (LOG_LIMITER.tryAcquire()) { + LOGGER.error("netty delay sender send failed,subject:{},group:{}", subject, groupName); + } + QMon.nettySendMessageFailCount(subject, groupName); + } + + private void monitorSendError(List records, BrokerGroupInfo group, int errorCode) { + records.parallelStream().forEach(record -> monitorSendError(record.getSubject(), group, errorCode)); + } + + private void monitorSendError(String subject, BrokerGroupInfo group, int errorCode) { + if (LOG_LIMITER.tryAcquire()) { + LOGGER.error("netty delay sender send error,subject:{},group:{},code:{}", subject, group, errorCode); + } + QMon.nettySendMessageFailCount(subject, group.getGroupName()); + } + + @Override + public void destroy() { + executorService.shutdown(); + try { + executorService.awaitTermination(5, TimeUnit.SECONDS); + } catch (InterruptedException e) { + LOGGER.error("Shutdown nettySenderExecutorService interrupted."); + } + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + SenderGroup that = (SenderGroup) o; + return Objects.equals(groupInfo.get(), that.groupInfo.get()); + } + + @Override + public int hashCode() { + return Objects.hash(groupInfo.get()); + } + + @Override + public String toString() { + return "SenderGroup{" + + "groupInfo=" + groupInfo.get() + + '}'; + } + + public interface ResultHandler { + void success(List recordList, Set messageIds); + + void fail(List records); + } + + private static class SenderRejectExecutionHandler implements RejectedExecutionHandler { + private final SenderGroup groupInfo; + + SenderRejectExecutionHandler(SenderGroup groupInfo) { + this.groupInfo = groupInfo; + } + + @Override + public void rejectedExecution(Runnable r, ThreadPoolExecutor executor) { + LOGGER.error("sender group:{} was rejected", groupInfo); + throw new RejectedExecutionException("Task " + r.toString() + + " rejected from " + + executor.toString()); + } + } + +} diff --git a/qmq-delay-server/src/main/java/qunar/tc/qmq/delay/sender/SenderProcessor.java b/qmq-delay-server/src/main/java/qunar/tc/qmq/delay/sender/SenderProcessor.java new file mode 100644 index 00000000..3d86a071 --- /dev/null +++ b/qmq-delay-server/src/main/java/qunar/tc/qmq/delay/sender/SenderProcessor.java @@ -0,0 +1,173 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.delay.sender; + +import com.google.common.collect.Sets; +import io.netty.buffer.ByteBuf; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import qunar.tc.qmq.batch.BatchExecutor; +import qunar.tc.qmq.batch.Processor; +import qunar.tc.qmq.broker.BrokerService; +import qunar.tc.qmq.common.ClientType; +import qunar.tc.qmq.configuration.DynamicConfig; +import qunar.tc.qmq.delay.DelayLogFacade; +import qunar.tc.qmq.delay.ScheduleIndex; +import qunar.tc.qmq.delay.meta.BrokerRoleManager; +import qunar.tc.qmq.delay.store.model.DispatchLogRecord; +import qunar.tc.qmq.delay.store.model.ScheduleSetRecord; + +import java.util.List; +import java.util.Set; +import java.util.concurrent.TimeUnit; + +import static qunar.tc.qmq.delay.ScheduleIndex.buildIndex; + +/** + * @author xufeng.deng dennisdxf@gmail.com + * @since 2018-07-25 13:59 + */ +public class SenderProcessor implements DelayProcessor, Processor, SenderGroup.ResultHandler { + private static final Logger LOGGER = LoggerFactory.getLogger(SenderProcessor.class); + + private static final long DEFAULT_SEND_WAIT_TIME = 1; + private static final int DEFAULT_SEND_THREAD = 4; + private static final int MAX_QUEUE_SIZE = 10000; + private static final int BATCH_SIZE = 30; + + private final DynamicConfig config; + private final SenderExecutor senderExecutor; + private final BrokerService brokerService; + private final DelayLogFacade facade; + + private BatchExecutor batchExecutor; + + private long sendWaitTime = DEFAULT_SEND_WAIT_TIME; + + public SenderProcessor(final DelayLogFacade facade, final BrokerService brokerService, final Sender sender, final DynamicConfig config) { + this.brokerService = brokerService; + this.senderExecutor = new SenderExecutor(sender, config); + this.facade = facade; + this.config = config; + } + + @Override + public void init() { + this.batchExecutor = new BatchExecutor<>("delay-sender", BATCH_SIZE, this, DEFAULT_SEND_THREAD); + this.batchExecutor.setQueueSize(MAX_QUEUE_SIZE); + config.addListener(conf -> { + this.batchExecutor.setThreads(conf.getInt("delay.send.batch.thread.size", DEFAULT_SEND_THREAD)); + this.sendWaitTime = conf.getLong("delay.send.wait.time", DEFAULT_SEND_WAIT_TIME); + }); + this.batchExecutor.init(); + } + + @Override + public void send(ByteBuf index) { + if (!BrokerRoleManager.isDelayMaster()) { + ScheduleIndex.release(index); + return; + } + + boolean add; + try { + long waitTime = Math.abs(sendWaitTime); + if (waitTime > 0) { + add = batchExecutor.addItem(index, waitTime, TimeUnit.MINUTES); + } else { + add = batchExecutor.addItem(index); + } + } catch (InterruptedException e) { + return; + } + if (!add) { + reject(index); + } + } + + @Override + public void process(List pureRecords) { + if (pureRecords == null || pureRecords.isEmpty()) { + return; + } + + List records = null; + try { + records = facade.recoverLogRecord(pureRecords); + senderExecutor.execute(records, this, brokerService); + } catch (Exception e) { + LOGGER.error("send message failed,messageSize:{} will retry", pureRecords.size(), e); + retry(records); + } + } + + private void reject(ByteBuf record) { + send(record); + } + + private void success(ScheduleSetRecord record) { + facade.appendDispatchLog(new DispatchLogRecord(record.getSubject(), record.getMessageId(), record.getScheduleTime(), record.getSequence())); + } + + private void retry(List records, Set messageIds) { + final Set refreshSubject = Sets.newHashSet(); + for (ScheduleSetRecord record : records) { + if (messageIds.contains(record.getMessageId())) { + refresh(record, refreshSubject); + send(buildIndex(record.getScheduleTime(), record.getStartWroteOffset(), record.getRecordSize(), record.getSequence())); + continue; + } + success(record); + } + } + + private void retry(List records) { + if (null == records || records.isEmpty()) { + return; + } + + final Set refreshSubject = Sets.newHashSet(); + for (ScheduleSetRecord record : records) { + refresh(record, refreshSubject); + send(buildIndex(record.getScheduleTime(), record.getStartWroteOffset(), record.getRecordSize(), record.getSequence())); + } + } + + private void refresh(ScheduleSetRecord record, Set refreshSubject) { + boolean refresh = !refreshSubject.contains(record.getSubject()); + if (refresh) { + brokerService.refresh(ClientType.PRODUCER, record.getSubject()); + refreshSubject.add(record.getSubject()); + } + } + + @Override + public void success(List recordList, Set messageIds) { + retry(recordList, messageIds); + } + + @Override + public void fail(List records) { + retry(records); + } + + @Override + public void destroy() { + batchExecutor.destroy(); + senderExecutor.destroy(); + } +} diff --git a/qmq-delay-server/src/main/java/qunar/tc/qmq/delay/startup/ServerWrapper.java b/qmq-delay-server/src/main/java/qunar/tc/qmq/delay/startup/ServerWrapper.java new file mode 100644 index 00000000..e5f9d667 --- /dev/null +++ b/qmq-delay-server/src/main/java/qunar/tc/qmq/delay/startup/ServerWrapper.java @@ -0,0 +1,186 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.delay.startup; + +import com.google.common.base.Preconditions; +import io.netty.buffer.ByteBuf; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import qunar.tc.qmq.broker.BrokerService; +import qunar.tc.qmq.broker.impl.BrokerServiceImpl; +import qunar.tc.qmq.common.Disposable; +import qunar.tc.qmq.concurrent.NamedThreadFactory; +import qunar.tc.qmq.configuration.BrokerConfig; +import qunar.tc.qmq.configuration.DynamicConfig; +import qunar.tc.qmq.configuration.DynamicConfigLoader; +import qunar.tc.qmq.delay.DefaultDelayLogFacade; +import qunar.tc.qmq.delay.DelayLogFacade; +import qunar.tc.qmq.delay.ScheduleIndex; +import qunar.tc.qmq.delay.config.DefaultStoreConfiguration; +import qunar.tc.qmq.delay.meta.BrokerRoleManager; +import qunar.tc.qmq.delay.receiver.ReceivedDelayMessageProcessor; +import qunar.tc.qmq.delay.receiver.Receiver; +import qunar.tc.qmq.delay.sender.NettySender; +import qunar.tc.qmq.delay.sender.Sender; +import qunar.tc.qmq.delay.sync.master.MasterSyncNettyServer; +import qunar.tc.qmq.delay.sync.slave.SlaveSynchronizer; +import qunar.tc.qmq.delay.wheel.WheelTickManager; +import qunar.tc.qmq.meta.BrokerRegisterService; +import qunar.tc.qmq.meta.BrokerRole; +import qunar.tc.qmq.meta.MetaServerLocator; +import qunar.tc.qmq.metainfoclient.MetaInfoService; +import qunar.tc.qmq.netty.DefaultConnectionEventHandler; +import qunar.tc.qmq.netty.NettyServer; +import qunar.tc.qmq.protocol.CommandCode; + +import java.util.concurrent.ExecutorService; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; + +/** + * @author xufeng.deng dennisdxf@gmail.com + * @since 2018-07-27 17:05 + */ +public class ServerWrapper implements Disposable { + private static final Logger LOGGER = LoggerFactory.getLogger(ServerWrapper.class); + + private static final String META_SERVER_ENDPOINT = "meta.server.endpoint"; + private static final String PORT_CONFIG = "broker.port"; + + private static final Integer DEFAULT_PORT = 20801; + + private ExecutorService receiveMessageExecutorService; + private BrokerRegisterService brokerRegisterService; + private ReceivedDelayMessageProcessor processor; + private MasterSyncNettyServer syncNettyServer; + private SlaveSynchronizer slaveSynchronizer; + private NettyServer nettyServer; + private DelayLogFacade facade; + private WheelTickManager wheelTickManager; + private Integer listenPort; + private Receiver receiver; + private DynamicConfig config; + + public void start() { + init(); + register(); + startServer(); + sync(); + online(); + } + + private void online() { + BrokerConfig.markAsWritable(); + brokerRegisterService.healthSwitch(true); + } + + private void init() { + this.config = DynamicConfigLoader.load("delay.properties"); + this.listenPort = config.getInt(PORT_CONFIG, DEFAULT_PORT); + + this.receiveMessageExecutorService = new ThreadPoolExecutor(1, 1, + 0L, TimeUnit.MILLISECONDS, new LinkedBlockingQueue<>(), new NamedThreadFactory("send-message-processor")); + + final Sender sender = new NettySender(); + final DefaultStoreConfiguration storeConfig = new DefaultStoreConfiguration(config); + this.facade = new DefaultDelayLogFacade(storeConfig, this::iterateCallback); + + MetaInfoService metaInfoService = new MetaInfoService(); + metaInfoService.setMetaServer(config.getString(META_SERVER_ENDPOINT)); + metaInfoService.init(); + BrokerService brokerService = new BrokerServiceImpl(metaInfoService); + this.wheelTickManager = new WheelTickManager(storeConfig, brokerService, facade, sender); + + this.receiver = new Receiver(config, facade); + + final MetaServerLocator metaServerLocator = new MetaServerLocator(config.getString(META_SERVER_ENDPOINT)); + this.brokerRegisterService = new BrokerRegisterService(listenPort, metaServerLocator); + + this.processor = new ReceivedDelayMessageProcessor(receiver); + } + + private boolean iterateCallback(final ByteBuf buf) { + long scheduleTime = ScheduleIndex.scheduleTime(buf); + long offset = ScheduleIndex.offset(buf); + if (wheelTickManager.canAdd(scheduleTime, offset)) { + wheelTickManager.addWHeel(buf); + return true; + } + + return false; + } + + private void register() { + this.brokerRegisterService.start(); + + Preconditions.checkState(BrokerConfig.getBrokerRole() != BrokerRole.STANDBY, "目前broker不允许被指定为standby模式"); + } + + private void sync() { + this.syncNettyServer = new MasterSyncNettyServer(config, facade); + this.syncNettyServer.registerSyncEvent(receiver); + this.syncNettyServer.start(); + + if (BrokerRoleManager.isDelaySlave()) { + this.slaveSynchronizer = new SlaveSynchronizer(BrokerConfig.getMasterAddress(), config, facade); + this.slaveSynchronizer.startSync(); + } + } + + private void startServer() { + wheelTickManager.start(); + facade.start(); + facade.blockUntilReplayDone(); + startNettyServer(); + } + + private void startNettyServer() { + this.nettyServer = new NettyServer("delay-broker", Runtime.getRuntime().availableProcessors(), listenPort, new DefaultConnectionEventHandler("delay-broker")); + this.nettyServer.registerProcessor(CommandCode.SEND_MESSAGE, processor, receiveMessageExecutorService); + this.nettyServer.start(); + } + + @Override + public void destroy() { + offline(); + receiveMessageExecutorService.shutdown(); + try { + receiveMessageExecutorService.awaitTermination(5, TimeUnit.SECONDS); + } catch (InterruptedException e) { + LOGGER.error("Shutdown receiveMessageExecutorService interrupted."); + } + brokerRegisterService.destroy(); + syncNettyServer.destroy(); + if (BrokerRoleManager.isDelaySlave()) { + slaveSynchronizer.destroy(); + } + nettyServer.destroy(); + facade.shutdown(); + wheelTickManager.shutdown(); + } + + private void offline() { + for (int i = 0; i < 3; ++i) { + try { + brokerRegisterService.healthSwitch(false); + } catch (Exception e) { + LOGGER.error("offline delay server failed", e); + } + } + } +} diff --git a/qmq-delay-server/src/main/java/qunar/tc/qmq/delay/store/DefaultDelaySegmentValidator.java b/qmq-delay-server/src/main/java/qunar/tc/qmq/delay/store/DefaultDelaySegmentValidator.java new file mode 100644 index 00000000..1dd41d47 --- /dev/null +++ b/qmq-delay-server/src/main/java/qunar/tc/qmq/delay/store/DefaultDelaySegmentValidator.java @@ -0,0 +1,33 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.delay.store; + +import qunar.tc.qmq.delay.store.log.DelaySegment; + +import java.io.IOException; + +/** + * @author xufeng.deng dennisdxf@gmail.com + * @since 2018-07-19 16:18 + */ +public class DefaultDelaySegmentValidator implements DelaySegmentValidator { + + @Override + public long validate(DelaySegment segment) throws IOException { + return segment.validate(); + } +} diff --git a/qmq-delay-server/src/main/java/qunar/tc/qmq/delay/store/DelaySegmentValidator.java b/qmq-delay-server/src/main/java/qunar/tc/qmq/delay/store/DelaySegmentValidator.java new file mode 100644 index 00000000..ccfbe8d5 --- /dev/null +++ b/qmq-delay-server/src/main/java/qunar/tc/qmq/delay/store/DelaySegmentValidator.java @@ -0,0 +1,30 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.delay.store; + +import qunar.tc.qmq.delay.store.log.DelaySegment; + +import java.io.IOException; + +/** + * @author xufeng.deng dennisdxf@gmail.com + * @since 2018-07-19 13:18 + */ +public interface DelaySegmentValidator { + + long validate(DelaySegment segment) throws IOException; +} diff --git a/qmq-delay-server/src/main/java/qunar/tc/qmq/delay/store/IterateOffsetManager.java b/qmq-delay-server/src/main/java/qunar/tc/qmq/delay/store/IterateOffsetManager.java new file mode 100644 index 00000000..be9d02cd --- /dev/null +++ b/qmq-delay-server/src/main/java/qunar/tc/qmq/delay/store/IterateOffsetManager.java @@ -0,0 +1,97 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.delay.store; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import qunar.tc.qmq.store.CheckpointStore; +import qunar.tc.qmq.store.FlushHook; +import qunar.tc.qmq.store.PeriodicFlushService; +import qunar.tc.qmq.store.Serde; + +import java.io.IOException; + +/** + * @author xufeng.deng dennisdxf@gmail.com + * @since 2018-07-19 17:31 + */ +public class IterateOffsetManager { + private static final String ITERATE_OFFSET_FILE = "message_log_iterate_checkpoint.json"; + private static final int DEFAULT_FLUSH_INTERVAL = 10 * 1000; + private static final ObjectMapper MAPPER = new ObjectMapper(); + + private final CheckpointStore offsetCheckpointStore; + private final FlushHook flushHook; + + private volatile long iterateOffset = 0; + + public IterateOffsetManager(String checkpointStorePath, FlushHook hook) { + this.offsetCheckpointStore = new CheckpointStore<>(checkpointStorePath, ITERATE_OFFSET_FILE, new IterateCheckpointSerde()); + Long offset = this.offsetCheckpointStore.loadCheckpoint(); + if (null != offset) { + this.iterateOffset = offset; + } + this.flushHook = hook; + } + + public synchronized void updateIterateOffset(long offset) { + if (offset > iterateOffset) { + this.iterateOffset = offset; + } + } + + public long getIterateOffset() { + return iterateOffset; + } + + public PeriodicFlushService.FlushProvider getFlushProvider() { + return new PeriodicFlushService.FlushProvider() { + + @Override + public int getInterval() { + return DEFAULT_FLUSH_INTERVAL; + } + + @Override + public void flush() { + flushHook.beforeFlush(); + offsetCheckpointStore.saveCheckpoint(iterateOffset); + } + }; + } + + private static final class IterateCheckpointSerde implements Serde { + + @Override + public byte[] toBytes(Long value) { + try { + return MAPPER.writeValueAsBytes(value); + } catch (JsonProcessingException e) { + throw new RuntimeException("serialize message log iterate offset checkpoint failed.", e); + } + } + + @Override + public Long fromBytes(byte[] data) { + try { + return MAPPER.readValue(data, Long.class); + } catch (IOException e) { + throw new RuntimeException("deserialize message log iterate offset checkpoint failed.", e); + } + } + } +} diff --git a/qmq-delay-server/src/main/java/qunar/tc/qmq/delay/store/ScheduleLogValidatorSupport.java b/qmq-delay-server/src/main/java/qunar/tc/qmq/delay/store/ScheduleLogValidatorSupport.java new file mode 100644 index 00000000..29c7f779 --- /dev/null +++ b/qmq-delay-server/src/main/java/qunar/tc/qmq/delay/store/ScheduleLogValidatorSupport.java @@ -0,0 +1,139 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.delay.store; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.base.Preconditions; +import com.google.common.io.Files; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import qunar.tc.qmq.delay.config.StoreConfiguration; +import qunar.tc.qmq.store.Serde; + +import java.io.File; +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; + +/** + * @author xufeng.deng dennisdxf@gmail.com + * @since 2018-08-16 14:59 + */ +public class ScheduleLogValidatorSupport { + private static final Logger LOGGER = LoggerFactory.getLogger(ScheduleLogValidatorSupport.class); + + public static final String SCHEDULE_OFFSET_CHECKPOINT = "schedule_offset_checkpoint.json"; + + private static final ScheduleOffsetSerde SERDE = new ScheduleOffsetSerde(); + private static final ObjectMapper MAPPER = new ObjectMapper(); + + private static ScheduleLogValidatorSupport SUPPORT; + + private final StoreConfiguration config; + + private ScheduleLogValidatorSupport(StoreConfiguration config) { + this.config = config; + } + + public static ScheduleLogValidatorSupport getSupport(StoreConfiguration config) { + if (null == SUPPORT) { + SUPPORT = new ScheduleLogValidatorSupport(config); + } + + return SUPPORT; + } + + public void saveScheduleOffsetCheckpoint(Map offsets) { + ensureDir(config.getScheduleOffsetCheckpointPath()); + final byte[] data = SERDE.toBytes(offsets); + Preconditions.checkState(data != null, "Serialized checkpoint data should not be null."); + if (data.length == 0) { + return; + } + + final File checkpoint = new File(config.getScheduleOffsetCheckpointPath(), SCHEDULE_OFFSET_CHECKPOINT); + try { + Files.write(data, checkpoint); + } catch (IOException e) { + LOGGER.error("write data into schedule checkpoint file failed. file={}", checkpoint, e); + throw new RuntimeException("write checkpoint data failed.", e); + } + } + + private void ensureDir(final String storePath) { + final File store = new File(storePath); + if (store.exists()) { + return; + } + + final boolean success = store.mkdirs(); + if (!success) { + throw new RuntimeException("Failed create path " + storePath); + } + LOGGER.info("Create checkpoint store {} success.", storePath); + } + + public Map loadScheduleOffsetCheckpoint() { + File file = new File(config.getScheduleOffsetCheckpointPath(), SCHEDULE_OFFSET_CHECKPOINT); + if (!file.exists()) { + return new HashMap<>(0); + } + + try { + final byte[] data = Files.toByteArray(file); + if (data != null && data.length == 0) { + if (!file.delete()) throw new RuntimeException("remove checkpoint error. filename=" + file); + return new HashMap<>(0); + } + Map offsets = SERDE.fromBytes(data); + if (null == offsets || !file.delete()) { + throw new RuntimeException("Load checkpoint error. filename=" + file); + } + + return offsets; + } catch (IOException e) { + LOGGER.error("Load checkpoint file failed.", e); + } + + throw new RuntimeException("Load checkpoint failed. filename=" + file); + } + + private static class ScheduleOffsetSerde implements Serde> { + + @Override + public byte[] toBytes(Map value) { + try { + return MAPPER.writeValueAsBytes(value); + } catch (JsonProcessingException e) { + throw new RuntimeException("serialize schedule offset failed.", e); + } + } + + @Override + public Map fromBytes(byte[] data) { + try { + return MAPPER.readValue(data, new TypeReference>() { + }); + } catch (IOException e) { + throw new RuntimeException("deserialize schedule offset checkpoint failed.", e); + } + } + } + +} diff --git a/qmq-delay-server/src/main/java/qunar/tc/qmq/delay/store/VisitorAccessor.java b/qmq-delay-server/src/main/java/qunar/tc/qmq/delay/store/VisitorAccessor.java new file mode 100644 index 00000000..4a5113f0 --- /dev/null +++ b/qmq-delay-server/src/main/java/qunar/tc/qmq/delay/store/VisitorAccessor.java @@ -0,0 +1,33 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.delay.store; + + +import qunar.tc.qmq.delay.store.model.LogRecord; +import qunar.tc.qmq.delay.store.visitor.LogVisitor; + +/** + * @author xufeng.deng dennisdxf@gmail.com + * @since 2018-07-11 19:52 + */ +public interface VisitorAccessor { + + LogVisitor newVisitor(T key); + + long getMaxOffset(); + +} diff --git a/qmq-delay-server/src/main/java/qunar/tc/qmq/delay/store/appender/DispatchLogAppender.java b/qmq-delay-server/src/main/java/qunar/tc/qmq/delay/store/appender/DispatchLogAppender.java new file mode 100644 index 00000000..90f44be6 --- /dev/null +++ b/qmq-delay-server/src/main/java/qunar/tc/qmq/delay/store/appender/DispatchLogAppender.java @@ -0,0 +1,52 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.delay.store.appender; + +import qunar.tc.qmq.delay.store.model.AppendRecordResult; +import qunar.tc.qmq.delay.store.model.LogRecord; +import qunar.tc.qmq.store.AppendMessageStatus; + +import java.nio.ByteBuffer; +import java.util.concurrent.locks.ReentrantLock; + +/** + * @author xufeng.deng dennisdxf@gmail.com + * @since 2018-07-19 16:43 + */ +public class DispatchLogAppender implements LogAppender { + private final ByteBuffer workingBuffer = ByteBuffer.allocate(Long.BYTES); + private final ReentrantLock lock = new ReentrantLock(); + + @Override + public AppendRecordResult appendLog(LogRecord log) { + workingBuffer.clear(); + workingBuffer.putLong(log.getSequence()); + workingBuffer.flip(); + + return new AppendRecordResult<>(AppendMessageStatus.SUCCESS, 0, Long.BYTES, workingBuffer,true); + } + + @Override + public void lockAppender() { + lock.lock(); + } + + @Override + public void unlockAppender() { + lock.unlock(); + } +} diff --git a/qmq-delay-server/src/main/java/qunar/tc/qmq/delay/store/appender/LogAppender.java b/qmq-delay-server/src/main/java/qunar/tc/qmq/delay/store/appender/LogAppender.java new file mode 100644 index 00000000..cebd487d --- /dev/null +++ b/qmq-delay-server/src/main/java/qunar/tc/qmq/delay/store/appender/LogAppender.java @@ -0,0 +1,34 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.delay.store.appender; + + +import qunar.tc.qmq.delay.store.model.AppendRecordResult; + +/** + * @author xufeng.deng dennisdxf@gmail.com + * @since 2018-07-13 17:34 + */ +public interface LogAppender { + + AppendRecordResult appendLog(final T log); + + void lockAppender(); + + void unlockAppender(); + +} diff --git a/qmq-delay-server/src/main/java/qunar/tc/qmq/delay/store/appender/ScheduleSetAppender.java b/qmq-delay-server/src/main/java/qunar/tc/qmq/delay/store/appender/ScheduleSetAppender.java new file mode 100644 index 00000000..c6790dca --- /dev/null +++ b/qmq-delay-server/src/main/java/qunar/tc/qmq/delay/store/appender/ScheduleSetAppender.java @@ -0,0 +1,84 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.delay.store.appender; + +import qunar.tc.qmq.delay.store.model.AppendRecordResult; +import qunar.tc.qmq.delay.store.model.LogRecord; +import qunar.tc.qmq.delay.store.model.ScheduleSetSequence; +import qunar.tc.qmq.store.AppendMessageStatus; + +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; +import java.util.concurrent.locks.ReentrantLock; + +/** + * @author xufeng.deng dennisdxf@gmail.com + * @since 2018-07-19 15:21 + */ +public class ScheduleSetAppender implements LogAppender { + + private final ByteBuffer workingBuffer; + private final ReentrantLock lock = new ReentrantLock(); + + public ScheduleSetAppender(int singleMessageSize) { + this.workingBuffer = ByteBuffer.allocate(singleMessageSize); + } + + @Override + public AppendRecordResult appendLog(LogRecord log) { + workingBuffer.clear(); + workingBuffer.flip(); + final byte[] subjectBytes = log.getSubject().getBytes(StandardCharsets.UTF_8); + final byte[] messageIdBytes = log.getMessageId().getBytes(StandardCharsets.UTF_8); + int recordSize = getRecordSize(log, subjectBytes.length, messageIdBytes.length); + workingBuffer.limit(recordSize); + + long scheduleTime = log.getScheduleTime(); + long sequence = log.getSequence(); + workingBuffer.putLong(scheduleTime); + workingBuffer.putLong(sequence); + workingBuffer.putInt(log.getPayloadSize()); + workingBuffer.putInt(messageIdBytes.length); + workingBuffer.put(messageIdBytes); + workingBuffer.putInt(subjectBytes.length); + workingBuffer.put(subjectBytes); + workingBuffer.put(log.getRecord()); + workingBuffer.flip(); + ScheduleSetSequence record = new ScheduleSetSequence(scheduleTime, sequence); + return new AppendRecordResult<>(AppendMessageStatus.SUCCESS, 0, recordSize, workingBuffer, record); + } + + private int getRecordSize(LogRecord record, int subject, int messageId) { + return 8 + 8 + + 4 + + 4 + + 4 + + subject + + messageId + + record.getPayloadSize(); + } + + @Override + public void lockAppender() { + lock.lock(); + } + + @Override + public void unlockAppender() { + lock.unlock(); + } +} diff --git a/qmq-delay-server/src/main/java/qunar/tc/qmq/delay/store/log/AbstractDelayLog.java b/qmq-delay-server/src/main/java/qunar/tc/qmq/delay/store/log/AbstractDelayLog.java new file mode 100644 index 00000000..92b9032f --- /dev/null +++ b/qmq-delay-server/src/main/java/qunar/tc/qmq/delay/store/log/AbstractDelayLog.java @@ -0,0 +1,63 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.delay.store.log; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import qunar.tc.qmq.delay.store.model.AppendLogResult; +import qunar.tc.qmq.delay.store.model.LogRecord; +import qunar.tc.qmq.delay.store.model.RecordResult; +import qunar.tc.qmq.protocol.producer.MessageProducerCode; +import qunar.tc.qmq.store.PutMessageStatus; + +/** + * @author xufeng.deng dennisdxf@gmail.com + * @since 2018-07-19 12:28 + */ +public abstract class AbstractDelayLog implements Log, LogRecord> { + private static final Logger LOGGER = LoggerFactory.getLogger(AbstractDelayLog.class); + + SegmentContainer, LogRecord> container; + + AbstractDelayLog(SegmentContainer, LogRecord> container) { + this.container = container; + } + + @Override + public AppendLogResult> append(LogRecord record) { + String subject = record.getSubject(); + RecordResult result = container.append(record); + PutMessageStatus status = result.getStatus(); + if (PutMessageStatus.SUCCESS != status) { + LOGGER.error("appendMessageLog schedule set file error,subject:{},status:{}", subject, status.name()); + return new AppendLogResult<>(MessageProducerCode.STORE_ERROR, status.name(), null); + } + + return new AppendLogResult<>(MessageProducerCode.SUCCESS, status.name(), result); + } + + @Override + public boolean clean(Long key) { + return container.clean(key); + } + + @Override + public void flush() { + container.flush(); + } + +} diff --git a/qmq-delay-server/src/main/java/qunar/tc/qmq/delay/store/log/AbstractDelaySegment.java b/qmq-delay-server/src/main/java/qunar/tc/qmq/delay/store/log/AbstractDelaySegment.java new file mode 100644 index 00000000..8570370d --- /dev/null +++ b/qmq-delay-server/src/main/java/qunar/tc/qmq/delay/store/log/AbstractDelaySegment.java @@ -0,0 +1,169 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.delay.store.log; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import qunar.tc.qmq.delay.store.appender.LogAppender; +import qunar.tc.qmq.delay.store.model.AppendRecordResult; +import qunar.tc.qmq.delay.store.model.LogRecord; +import qunar.tc.qmq.store.AppendMessageResult; +import qunar.tc.qmq.store.AppendMessageStatus; + +import java.io.File; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.io.RandomAccessFile; +import java.nio.channels.FileChannel; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicLong; + +/** + * @author xufeng.deng dennisdxf@gmail.com + * @since 2018-07-19 11:02 + */ +public abstract class AbstractDelaySegment implements DelaySegment { + private static final Logger LOGGER = LoggerFactory.getLogger(AbstractDelaySegment.class); + + private final File file; + private final int segmentBaseOffset; + private final AtomicLong wrotePosition = new AtomicLong(0); + private final AtomicLong flushedPosition = new AtomicLong(0); + private final AtomicBoolean needFlush = new AtomicBoolean(true); + + final String fileName; + + FileChannel fileChannel; + + AbstractDelaySegment(File file) throws IOException { + this.file = file; + this.fileName = file.getAbsolutePath(); + this.segmentBaseOffset = Integer.parseInt(file.getName()); + boolean success = false; + try { + fileChannel = new RandomAccessFile(file, "rw").getChannel(); + success = true; + } catch (FileNotFoundException e) { + LOGGER.error("create file channel failed. file: {}", fileName, e); + throw e; + } finally { + if (!success && null != fileChannel) { + fileChannel.close(); + } + } + } + + @Override + public AppendMessageResult append(LogRecord log, LogAppender appender) { + appender.lockAppender(); + try { + long currentPos = wrotePosition.get(); + AppendRecordResult result = appender.appendLog(log); + + AppendMessageStatus status = result.getStatus(); + if (AppendMessageStatus.SUCCESS != result.getStatus()) { + LOGGER.error("appendMessageLog delay segment error,subject:{},status:{},segment file:{}", log.getSubject(), status.name(), fileName); + return new AppendMessageResult<>(AppendMessageStatus.UNKNOWN_ERROR, -1, -1); + } + + int wroteBytes = result.getWroteBytes(); + + // This method would not modify this channel's position. + int writes = fileChannel.write(result.getBuffer(), currentPos); + if (writes != wroteBytes) { + LOGGER.error("appendMessageLog delay segment error,appendMessageLog size is ex,segment file:{},record size:{},written:{}", fileName, wroteBytes, writes); + return new AppendMessageResult<>(AppendMessageStatus.APPEND_FAILED, -1, -1); + } + + long channelPosition = wrotePosition.addAndGet(wroteBytes); + this.needFlush.set(true); + fileChannel.position(channelPosition); + return new AppendMessageResult<>(AppendMessageStatus.SUCCESS, currentPos, wroteBytes, result.getAdditional()); + } catch (Exception e) { + LOGGER.error("appendMessageLog delay segment error,io ex,segment file:{}", fileName, e); + return new AppendMessageResult<>(AppendMessageStatus.UNKNOWN_ERROR, -1, -1); + } finally { + appender.unlockAppender(); + } + } + + @Override + public void setWrotePosition(long position) { + wrotePosition.set(position); + } + + @Override + public long getWrotePosition() { + return wrotePosition.get(); + } + + @Override + public long getFlushedPosition() { + return flushedPosition.get(); + } + + @Override + public void setFlushedPosition(long position) { + flushedPosition.set(position); + } + + @Override + public int getSegmentBaseOffset() { + return segmentBaseOffset; + } + + @Override + public boolean destroy() { + close(); + return file.delete(); + } + + private void close() { + try { + fileChannel.close(); + } catch (Exception e) { + LOGGER.error("close file channel failed. file: {}", fileName, e); + } + } + + @Override + public long flush() { + if (!this.needFlush.get()) { + return getFlushedPosition(); + } + + long value = wrotePosition.get(); + try { + fileChannel.force(true); + } catch (Throwable e) { + LOGGER.error("Error occurred when flush data to disk.", e); + return getFlushedPosition(); + } + flushedPosition.set(value); + this.needFlush.set(false); + + return getFlushedPosition(); + } + + @Override + public String toString() { + return "DelaySegment{" + + "file=" + fileName + + "}"; + } + +} diff --git a/qmq-delay-server/src/main/java/qunar/tc/qmq/delay/store/log/AbstractDelaySegmentContainer.java b/qmq-delay-server/src/main/java/qunar/tc/qmq/delay/store/log/AbstractDelaySegmentContainer.java new file mode 100644 index 00000000..4c577a08 --- /dev/null +++ b/qmq-delay-server/src/main/java/qunar/tc/qmq/delay/store/log/AbstractDelaySegmentContainer.java @@ -0,0 +1,134 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.delay.store.log; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import qunar.tc.qmq.delay.store.DelaySegmentValidator; +import qunar.tc.qmq.delay.store.appender.LogAppender; +import qunar.tc.qmq.delay.store.model.LogRecord; +import qunar.tc.qmq.delay.store.model.NopeRecordResult; +import qunar.tc.qmq.delay.store.model.RecordResult; +import qunar.tc.qmq.store.AppendMessageResult; +import qunar.tc.qmq.store.PutMessageStatus; + +import java.io.File; +import java.util.concurrent.ConcurrentSkipListMap; + +import static qunar.tc.qmq.delay.store.log.ScheduleOffsetResolver.resolveSegment; + + +/** + * @author xufeng.deng dennisdxf@gmail.com + * @since 2018-07-19 10:36 + */ +public abstract class AbstractDelaySegmentContainer implements SegmentContainer, LogRecord> { + private static final Logger LOGGER = LoggerFactory.getLogger(AbstractDelaySegmentContainer.class); + + File logDir; + + private final LogAppender appender; + + final ConcurrentSkipListMap> segments = new ConcurrentSkipListMap<>(); + + AbstractDelaySegmentContainer(File logDir, DelaySegmentValidator validator, LogAppender appender) { + this.logDir = logDir; + this.appender = appender; + createAndValidateLogDir(); + loadLogs(validator); + } + + protected abstract void loadLogs(DelaySegmentValidator validator); + + private void createAndValidateLogDir() { + if (!logDir.exists()) { + LOGGER.info("Log directory {} not found, try create it.", logDir.getAbsoluteFile()); + boolean created = logDir.mkdirs(); + if (!created) { + throw new RuntimeException("Failed to create log directory " + logDir.getAbsolutePath()); + } + } + + if (!logDir.isDirectory() || !logDir.canRead() || !logDir.canWrite()) { + throw new RuntimeException(logDir.getAbsolutePath() + " is not a readable log directory"); + } + } + + @Override + @SuppressWarnings("unchecked") + public RecordResult append(LogRecord record) { + long scheduleTime = record.getScheduleTime(); + DelaySegment segment = locateSegment(scheduleTime); + if (null == segment) { + segment = allocNewSegment(scheduleTime); + } + + if (null == segment) { + return new NopeRecordResult(PutMessageStatus.CREATE_MAPPED_FILE_FAILED); + } + + return retResult(segment.append(record, appender)); + } + + @Override + public boolean clean(Long key) { + if (segments.isEmpty()) return false; + if (segments.lastKey() < key) return false; + DelaySegment segment = segments.remove(Math.toIntExact(key)); + if (null == segment) { + LOGGER.error("clean delay segment log failed,segment:{}", logDir, key); + return false; + } + + if (!segment.destroy()) { + LOGGER.warn("remove delay segment failed.segment:{}", segment); + return false; + } + + LOGGER.info("remove delay segment success.segment:{}", segment); + return true; + } + + @Override + public void flush() { + for (DelaySegment segment : segments.values()) { + segment.flush(); + } + } + + protected abstract RecordResult retResult(AppendMessageResult result); + + DelaySegment locateSegment(long scheduleTime) { + int baseOffset = resolveSegment(scheduleTime); + return segments.get(baseOffset); + } + + private DelaySegment allocNewSegment(long offset) { + int baseOffset = resolveSegment(offset); + if (segments.containsKey(baseOffset)) { + return segments.get(baseOffset); + } + return allocSegment(baseOffset); + } + + int higherBaseOffset(int low) { + Integer next = segments.higherKey(low); + return next == null ? -1 : next; + } + + protected abstract DelaySegment allocSegment(int segmentBaseOffset); +} diff --git a/qmq-delay-server/src/main/java/qunar/tc/qmq/delay/store/log/DelaySegment.java b/qmq-delay-server/src/main/java/qunar/tc/qmq/delay/store/log/DelaySegment.java new file mode 100644 index 00000000..a8faaf46 --- /dev/null +++ b/qmq-delay-server/src/main/java/qunar/tc/qmq/delay/store/log/DelaySegment.java @@ -0,0 +1,48 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.delay.store.log; + +import qunar.tc.qmq.delay.store.appender.LogAppender; +import qunar.tc.qmq.delay.store.model.LogRecord; +import qunar.tc.qmq.store.AppendMessageResult; + +import java.io.IOException; + +/** + * @author xufeng.deng dennisdxf@gmail.com + * @since 2018-07-19 10:50 + */ +public interface DelaySegment { + + AppendMessageResult append(LogRecord log, LogAppender appender); + + void setWrotePosition(long position); + + long getWrotePosition(); + + void setFlushedPosition(long position); + + long getFlushedPosition(); + + int getSegmentBaseOffset(); + + long validate() throws IOException; + + boolean destroy(); + + long flush(); +} diff --git a/qmq-delay-server/src/main/java/qunar/tc/qmq/delay/store/log/DispatchLog.java b/qmq-delay-server/src/main/java/qunar/tc/qmq/delay/store/log/DispatchLog.java new file mode 100644 index 00000000..bfae33ce --- /dev/null +++ b/qmq-delay-server/src/main/java/qunar/tc/qmq/delay/store/log/DispatchLog.java @@ -0,0 +1,91 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.delay.store.log; + +import qunar.tc.qmq.delay.cleaner.LogCleaner; +import qunar.tc.qmq.delay.config.StoreConfiguration; +import qunar.tc.qmq.delay.store.DefaultDelaySegmentValidator; +import qunar.tc.qmq.delay.store.appender.DispatchLogAppender; +import qunar.tc.qmq.store.PeriodicFlushService; +import qunar.tc.qmq.store.SegmentBuffer; +import qunar.tc.qmq.sync.DelaySyncRequest; + +import java.io.File; +import java.nio.ByteBuffer; + +/** + * @author xufeng.deng dennisdxf@gmail.com + * @since 2018-07-19 16:36 + */ +public class DispatchLog extends AbstractDelayLog { + /** + * message log flush interval,500ms + */ + private static final int DEFAULT_FLUSH_INTERVAL = 500; + + public DispatchLog(StoreConfiguration storeConfiguration) { + super(new DispatchLogSegmentContainer(storeConfiguration, + new File(storeConfiguration.getDispatchLogStorePath()) + , new DefaultDelaySegmentValidator(), new DispatchLogAppender())); + } + + public PeriodicFlushService.FlushProvider getProvider() { + return new PeriodicFlushService.FlushProvider() { + @Override + public int getInterval() { + return DEFAULT_FLUSH_INTERVAL; + } + + @Override + public void flush() { + DispatchLog.this.flush(); + } + }; + } + + public DispatchLogSegment latestSegment() { + return ((DispatchLogSegmentContainer) container).latestSegment(); + } + + public void clean(LogCleaner.CleanHook hook) { + ((DispatchLogSegmentContainer) container).clean(hook); + } + + public SegmentBuffer getDispatchLogData(int segmentBaseOffset, long dispatchLogOffset) { + return ((DispatchLogSegmentContainer) container).getDispatchData(segmentBaseOffset, dispatchLogOffset); + } + + public long getMaxOffset(int dispatchSegmentBaseOffset) { + return ((DispatchLogSegmentContainer) container).getMaxOffset(dispatchSegmentBaseOffset); + } + + public DelaySyncRequest.DispatchLogSyncRequest getSyncMaxRequest() { + return ((DispatchLogSegmentContainer) container).getSyncMaxRequest(); + } + + public boolean appendData(long startOffset, int baseOffset, ByteBuffer body) { + return ((DispatchLogSegmentContainer) container).appendData(startOffset, baseOffset, body); + } + + public DispatchLogSegment lowerSegment(int latestOffset) { + return ((DispatchLogSegmentContainer) container).lowerSegment(latestOffset); + } + + public int higherBaseOffset(int low) { + return ((DispatchLogSegmentContainer) container).higherBaseOffset(low); + } +} diff --git a/qmq-delay-server/src/main/java/qunar/tc/qmq/delay/store/log/DispatchLogSegment.java b/qmq-delay-server/src/main/java/qunar/tc/qmq/delay/store/log/DispatchLogSegment.java new file mode 100644 index 00000000..2b9c9217 --- /dev/null +++ b/qmq-delay-server/src/main/java/qunar/tc/qmq/delay/store/log/DispatchLogSegment.java @@ -0,0 +1,108 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.delay.store.log; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import qunar.tc.qmq.delay.base.SegmentBufferExtend; +import qunar.tc.qmq.delay.store.visitor.DispatchLogVisitor; +import qunar.tc.qmq.delay.store.visitor.LogVisitor; + +import java.io.File; +import java.io.IOException; +import java.nio.ByteBuffer; + +/** + * @author xufeng.deng dennisdxf@gmail.com + * @since 2018-07-19 16:41 + */ +public class DispatchLogSegment extends AbstractDelaySegment { + private static final Logger LOGGER = LoggerFactory.getLogger(DispatchLogSegment.class); + + DispatchLogSegment(File file) throws IOException { + super(file); + } + + @Override + public long validate() throws IOException { + long size = this.fileChannel.size(); + long invalidateBytes = size % Long.BYTES; + return size - invalidateBytes; + } + + public LogVisitor newVisitor(long from) { + return new DispatchLogVisitor(from, fileChannel); + } + + SegmentBufferExtend selectSegmentBuffer(long offset) { + long wrotePosition = getWrotePosition(); + if (wrotePosition == 0) { + return new SegmentBufferExtend(0, null, 0, getSegmentBaseOffset(), null); + } + + if (offset < wrotePosition && offset >= 0) { + int size = (int) (wrotePosition - offset); + final ByteBuffer buffer = ByteBuffer.allocate(size); + try { + int bytes = fileChannel.read(buffer, offset); + if (bytes < size) { + LOGGER.error("select dispatch log incomplete data to log segment,{}-{}-{}, {} -> {}", getSegmentBaseOffset(), wrotePosition, offset, bytes, size); + return null; + } + + buffer.flip(); + buffer.limit(bytes); + return new SegmentBufferExtend(offset, buffer, bytes, getSegmentBaseOffset(), null); + } catch (Throwable e) { + LOGGER.error("select dispatch log data to log segment failed.", e); + } + } + + return null; + } + + boolean appendData(long startOffset, ByteBuffer body) { + long currentPos = getWrotePosition(); + int size = body.limit(); + if (startOffset != currentPos) { + return false; + } + + try { + fileChannel.position(currentPos); + fileChannel.write(body); + } catch (Throwable e) { + LOGGER.error("appendMessageLog data to log segment failed.", e); + return false; + } + + setWrotePosition(currentPos + size); + return true; + } + + void fillPreBlank(long untilWhere) { + setWrotePosition(untilWhere); + } + + public int entries() { + try { + return (int) (validate() / Long.BYTES); + } catch (Exception e) { + return 0; + } + } +} diff --git a/qmq-delay-server/src/main/java/qunar/tc/qmq/delay/store/log/DispatchLogSegmentContainer.java b/qmq-delay-server/src/main/java/qunar/tc/qmq/delay/store/log/DispatchLogSegmentContainer.java new file mode 100644 index 00000000..5813f7e8 --- /dev/null +++ b/qmq-delay-server/src/main/java/qunar/tc/qmq/delay/store/log/DispatchLogSegmentContainer.java @@ -0,0 +1,183 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.delay.store.log; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import qunar.tc.qmq.delay.cleaner.LogCleaner; +import qunar.tc.qmq.delay.config.StoreConfiguration; +import qunar.tc.qmq.delay.store.DelaySegmentValidator; +import qunar.tc.qmq.delay.store.appender.LogAppender; +import qunar.tc.qmq.delay.store.model.AppendDispatchRecordResult; +import qunar.tc.qmq.delay.store.model.LogRecord; +import qunar.tc.qmq.delay.store.model.RecordResult; +import qunar.tc.qmq.store.AppendMessageResult; +import qunar.tc.qmq.store.PutMessageStatus; +import qunar.tc.qmq.store.SegmentBuffer; +import qunar.tc.qmq.sync.DelaySyncRequest; + +import java.io.File; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.Map; + +import static qunar.tc.qmq.delay.store.log.ScheduleOffsetResolver.resolveSegment; + +/** + * @author xufeng.deng dennisdxf@gmail.com + * @since 2018-07-19 16:39 + */ +public class DispatchLogSegmentContainer extends AbstractDelaySegmentContainer { + private static final Logger LOGGER = LoggerFactory.getLogger(DispatchLogSegmentContainer.class); + + private final StoreConfiguration config; + + DispatchLogSegmentContainer(StoreConfiguration config, File logDir, DelaySegmentValidator validator, LogAppender appender) { + super(logDir, validator, appender); + this.config = config; + } + + @Override + protected void loadLogs(DelaySegmentValidator validator) { + LOGGER.info("Loading logs."); + File[] files = this.logDir.listFiles(); + if (files != null) { + for (final File file : files) { + if (file.getName().startsWith(".")) { + continue; + } + if (file.isDirectory()) { + continue; + } + + DelaySegment segment; + try { + segment = new DispatchLogSegment(file); + long size = validator.validate(segment); + segment.setWrotePosition(size); + segment.setFlushedPosition(size); + segments.put(segment.getSegmentBaseOffset(), segment); + } catch (IOException e) { + LOGGER.error("Load {} failed.", file.getAbsolutePath(), e); + } + } + } + LOGGER.info("Load logs done."); + } + + @Override + protected RecordResult retResult(AppendMessageResult result) { + switch (result.getStatus()) { + case SUCCESS: + return new AppendDispatchRecordResult(PutMessageStatus.SUCCESS, result); + default: + return new AppendDispatchRecordResult(PutMessageStatus.UNKNOWN_ERROR, result); + } + } + + @Override + protected DelaySegment allocSegment(int segmentBaseOffset) { + File nextSegmentFile = new File(logDir, String.valueOf(segmentBaseOffset)); + try { + DelaySegment logSegment = new DispatchLogSegment(nextSegmentFile); + segments.put(segmentBaseOffset, logSegment); + LOGGER.info("alloc new dispatch log segment file {}", ((DispatchLogSegment) logSegment).fileName); + return logSegment; + } catch (IOException e) { + LOGGER.error("Failed create new dispatch log segment file. file: {}", nextSegmentFile.getAbsolutePath(), e); + } + return null; + } + + DispatchLogSegment latestSegment() { + Map.Entry> entry = segments.lastEntry(); + if (null == entry) { + return null; + } + + return ((DispatchLogSegment) segments.lastEntry().getValue()); + } + + public void clean(LogCleaner.CleanHook hook) { + Integer deleteUntil = resolveSegment(System.currentTimeMillis() - config.getDispatchLogKeepTime()); + for (DelaySegment segment : segments.values()) { + if (segment.getSegmentBaseOffset() < deleteUntil) { + doClean(segment, hook); + } + } + } + + private void doClean(DelaySegment segment, LogCleaner.CleanHook hook) { + long segmentBaseOffset = segment.getSegmentBaseOffset(); + if (clean(segmentBaseOffset) && hook != null) { + hook.clean(segmentBaseOffset); + } + } + + SegmentBuffer getDispatchData(int segmentBaseOffset, long dispatchLogOffset) { + DispatchLogSegment segment = (DispatchLogSegment) segments.get(segmentBaseOffset); + if (null == segment) { + return null; + } + + return segment.selectSegmentBuffer(dispatchLogOffset); + } + + long getMaxOffset(int segmentOffset) { + DispatchLogSegment segment = (DispatchLogSegment) segments.get(segmentOffset); + if (null == segment) { + return 0; + } + + return segment.getWrotePosition(); + } + + DelaySyncRequest.DispatchLogSyncRequest getSyncMaxRequest() { + final DispatchLogSegment segment = latestSegment(); + if (segment == null) { + return null; + } + + int lastBaseOffset = -1; + long lastOffset = -1; + final DispatchLogSegment lastSegment = lowerSegment(segment.getSegmentBaseOffset()); + if (lastSegment != null) { + lastBaseOffset = lastSegment.getSegmentBaseOffset(); + lastOffset = lastSegment.getWrotePosition(); + } + + return new DelaySyncRequest.DispatchLogSyncRequest(segment.getSegmentBaseOffset(), segment.getWrotePosition(), lastBaseOffset, lastOffset); + } + + boolean appendData(long startOffset, int baseOffset, ByteBuffer body) { + DispatchLogSegment segment = (DispatchLogSegment) segments.get(baseOffset); + if (null == segment) { + segment = (DispatchLogSegment) allocSegment(baseOffset); + segment.fillPreBlank(startOffset); + } + + return segment.appendData(startOffset, body); + } + + DispatchLogSegment lowerSegment(int offset) { + Map.Entry> lowEntry = segments.lowerEntry(offset); + if (lowEntry == null) { + return null; + } + return (DispatchLogSegment) lowEntry.getValue(); + } +} diff --git a/qmq-delay-server/src/main/java/qunar/tc/qmq/delay/store/log/Log.java b/qmq-delay-server/src/main/java/qunar/tc/qmq/delay/store/log/Log.java new file mode 100644 index 00000000..65655de1 --- /dev/null +++ b/qmq-delay-server/src/main/java/qunar/tc/qmq/delay/store/log/Log.java @@ -0,0 +1,31 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.delay.store.log; + +import qunar.tc.qmq.delay.store.model.AppendLogResult; + +/** + * @author xufeng.deng dennisdxf@gmail.com + * @since 2018-07-19 9:28 + */ +public interface Log { + AppendLogResult append(T record); + + boolean clean(Long key); + + void flush(); +} diff --git a/qmq-delay-server/src/main/java/qunar/tc/qmq/delay/store/log/MessageLog.java b/qmq-delay-server/src/main/java/qunar/tc/qmq/delay/store/log/MessageLog.java new file mode 100644 index 00000000..a9051efc --- /dev/null +++ b/qmq-delay-server/src/main/java/qunar/tc/qmq/delay/store/log/MessageLog.java @@ -0,0 +1,165 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.delay.store.log; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import qunar.tc.qmq.base.MessageHeader; +import qunar.tc.qmq.delay.config.StoreConfiguration; +import qunar.tc.qmq.delay.monitor.QMon; +import qunar.tc.qmq.delay.store.VisitorAccessor; +import qunar.tc.qmq.delay.store.model.AppendLogResult; +import qunar.tc.qmq.delay.store.model.AppendMessageRecordResult; +import qunar.tc.qmq.delay.store.model.LogRecord; +import qunar.tc.qmq.delay.store.model.RawMessageExtend; +import qunar.tc.qmq.delay.store.visitor.LogVisitor; +import qunar.tc.qmq.protocol.producer.MessageProducerCode; +import qunar.tc.qmq.store.PeriodicFlushService; +import qunar.tc.qmq.store.PutMessageStatus; +import qunar.tc.qmq.store.SegmentBuffer; + +import java.nio.ByteBuffer; + +/** + * @author xufeng.deng dennisdxf@gmail.com + * @since 2018-07-19 9:29 + */ +public class MessageLog implements Log, VisitorAccessor { + private static final Logger LOGGER = LoggerFactory.getLogger(MessageLog.class); + + private static final int DEFAULT_FLUSH_INTERVAL = 500; + + private final SegmentContainer container; + + public MessageLog(StoreConfiguration config) { + this.container = new MessageSegmentContainer(config); + } + + @Override + public AppendLogResult append(RawMessageExtend record) { + long start = System.currentTimeMillis(); + MessageHeader header = record.getHeader(); + String messageId = header.getMessageId(); + String subject = header.getSubject(); + + AppendMessageRecordResult recordResult; + try { + recordResult = container.append(record); + PutMessageStatus status = recordResult.getStatus(); + if (PutMessageStatus.MESSAGE_ILLEGAL == status) { + LOGGER.error("appendMessageLog message log error,log:{} {}", subject, messageId); + appendFailedByMessageIllegal(subject); + return new AppendLogResult<>(MessageProducerCode.SUCCESS, status.name(), new MessageLog.MessageRecordMeta(messageId, -1)); + } + + if (PutMessageStatus.SUCCESS != status) { + LOGGER.error("appendMessageLog message log error,log:{} {},status:{}", subject, messageId, status.name()); + appendFailed(subject); + return new AppendLogResult<>(MessageProducerCode.STORE_ERROR, status.name(), new MessageLog.MessageRecordMeta(messageId, -1)); + } + + return new AppendLogResult<>(MessageProducerCode.SUCCESS, status.name(), new MessageLog.MessageRecordMeta(messageId, recordResult.getResult().getAdditional())); + } catch (Throwable e) { + LOGGER.error("appendMessageLog message log error,log:{} {}, msg:{}", subject, messageId, record, e); + appendFailed(subject); + return new AppendLogResult<>(MessageProducerCode.STORE_ERROR, "", new MessageLog.MessageRecordMeta(messageId, -1)); + } finally { + appendTimer(subject, System.currentTimeMillis() - start); + } + } + + private void appendFailedByMessageIllegal(String subject) { + QMon.appendFailedByMessageIllegal(subject); + } + + @Override + public boolean clean(Long key) { + return container.clean(key); + } + + @Override + public void flush() { + container.flush(); + } + + @Override + public LogVisitor newVisitor(Long key) { + return ((MessageSegmentContainer) container).newLogVisitor(key); + } + + @Override + public long getMaxOffset() { + return ((MessageSegmentContainer) container).getMaxOffset(); + } + + public PeriodicFlushService.FlushProvider getProvider() { + return new PeriodicFlushService.FlushProvider() { + @Override + public int getInterval() { + return DEFAULT_FLUSH_INTERVAL; + } + + @Override + public void flush() { + MessageLog.this.flush(); + } + }; + } + + public void clean() { + ((MessageSegmentContainer) container).clean(); + } + + private static void appendFailed(String subject) { + QMon.appendFailed(subject); + } + + private static void appendTimer(String subject, long cost) { + QMon.appendTimer(subject, cost); + } + + public long getMinOffset() { + return ((MessageSegmentContainer) container).getMinOffset(); + } + + public SegmentBuffer getMessageLogData(long startSyncOffset) { + return ((MessageSegmentContainer) container).getMessageData(startSyncOffset); + } + + public boolean appendData(long startOffset, ByteBuffer buffer) { + return ((MessageSegmentContainer) container).appendData(startOffset, buffer); + } + + public static class MessageRecordMeta { + private String messageId; + + private long messageOffset; + + MessageRecordMeta(String messageId, long messageOffset) { + this.messageId = messageId; + this.messageOffset = messageOffset; + } + + public String getMessageId() { + return messageId; + } + + public long getMessageOffset() { + return messageOffset; + } + } +} diff --git a/qmq-delay-server/src/main/java/qunar/tc/qmq/delay/store/log/MessageSegmentContainer.java b/qmq-delay-server/src/main/java/qunar/tc/qmq/delay/store/log/MessageSegmentContainer.java new file mode 100644 index 00000000..11f1f4d4 --- /dev/null +++ b/qmq-delay-server/src/main/java/qunar/tc/qmq/delay/store/log/MessageSegmentContainer.java @@ -0,0 +1,407 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.delay.store.log; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import qunar.tc.qmq.delay.config.StoreConfiguration; +import qunar.tc.qmq.delay.store.model.AppendMessageRecordResult; +import qunar.tc.qmq.delay.store.model.LogRecord; +import qunar.tc.qmq.delay.store.model.MessageLogAttrEnum; +import qunar.tc.qmq.delay.store.model.RawMessageExtend; +import qunar.tc.qmq.delay.store.visitor.DelayMessageLogVisitor; +import qunar.tc.qmq.delay.store.visitor.LogVisitor; +import qunar.tc.qmq.store.*; + +import java.io.File; +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; +import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.locks.ReentrantLock; + +import static qunar.tc.qmq.delay.store.model.MessageLogAttrEnum.*; +import static qunar.tc.qmq.store.MagicCode.MESSAGE_LOG_MAGIC_V1; +import static qunar.tc.qmq.store.MagicCode.MESSAGE_LOG_MAGIC_V2; + +/** + * @author xufeng.deng dennisdxf@gmail.com + * @since 2018-07-19 10:06 + */ +public class MessageSegmentContainer implements SegmentContainer { + private static final Logger LOGGER = LoggerFactory.getLogger(MessageSegmentContainer.class); + + private static final int MIN_RECORD_BYTES = 13; + + private final LogManager logManager; + private final MessageAppender messageAppender; + private final StoreConfiguration config; + private final AtomicLong sequence = new AtomicLong(0); + + MessageSegmentContainer(StoreConfiguration config) { + this.config = config; + this.messageAppender = new MessageSegmentContainer.DelayRawMessageAppender(); + this.logManager = new LogManager(new File(config.getMessageLogStorePath()) + , config.getMessageLogSegmentFileSize() + , new StorageConfigImpl(config.getConfig()) + , new MessageLogSegmentValidator()); + recoverSequence(); + } + + private void recoverSequence() { + LogSegment logSegment = logManager.latestSegment(); + if (null == logSegment) { + LOGGER.warn("recover sequence happened null segment"); + return; + } + validateSequence(logSegment); + } + + private void validateSequence(LogSegment logSegment) { + LOGGER.info("validate logs."); + final int fileSize = logSegment.getFileSize(); + final ByteBuffer buffer = logSegment.sliceByteBuffer(); + + int position = 0; + while (true) { + if (position == fileSize) { + break; + } + + final int result = doValidateSequence(buffer, sequence.get()); + if (result == -1) { + break; + } else { + position += result; + } + } + + LOGGER.info("validate logs done."); + } + + private int doValidateSequence(final ByteBuffer buffer, long lastSequence) { + try { + final int magic = buffer.getInt(); + if (!MagicCodeSupport.isValidMessageLogMagicCode(magic)) { + return -1; + } + + final byte attributes = buffer.get(); + buffer.getLong(); + if (attributes == ATTR_SKIP_RECORD.getCode()) { + return buffer.getInt(); + } else if (attributes == ATTR_MESSAGE_RECORD.getCode()) { + return resolveSequence(buffer, magic, lastSequence); + } else { + return -1; + } + } catch (Exception e) { + LOGGER.error("message log recover resolve sequence error", e); + return -1; + } + } + + private int resolveSequence(final ByteBuffer buffer, final int magic, final long lastSequence) { + try { + buffer.getLong(); + long nextSequence = buffer.getLong(); + final int messageIdSize = buffer.getInt(); + buffer.position(buffer.position() + messageIdSize); + final int subjectSize = buffer.getInt(); + buffer.position(buffer.position() + subjectSize); + if (magic >= MESSAGE_LOG_MAGIC_V2) { + buffer.getLong(); + final int payloadSize = buffer.getInt(); + buffer.position(buffer.position() + payloadSize); + sequence.set(Math.max(nextSequence, lastSequence)); + return recordSizeWithCrc(messageIdSize, subjectSize, payloadSize); + } else { + final int payloadSize = buffer.getInt(); + buffer.position(buffer.position() + payloadSize); + sequence.set(Math.max(nextSequence, lastSequence)); + return recordSize(messageIdSize, subjectSize, payloadSize); + } + } catch (Exception e) { + LOGGER.error("message log recover complete record, resolve sequence error", e); + return -1; + } + } + + @Override + public AppendMessageRecordResult append(RawMessageExtend record) { + AppendMessageResult result; + LogSegment segment = logManager.latestSegment(); + if (null == segment) { + segment = logManager.allocNextSegment(); + } + + if (null == segment) { + return new AppendMessageRecordResult(PutMessageStatus.CREATE_MAPPED_FILE_FAILED, null); + } + + result = segment.append(record, messageAppender); + switch (result.getStatus()) { + case MESSAGE_SIZE_EXCEEDED: + return new AppendMessageRecordResult(PutMessageStatus.MESSAGE_ILLEGAL, null); + case END_OF_FILE: + if (null == logManager.allocNextSegment()) { + return new AppendMessageRecordResult(PutMessageStatus.CREATE_MAPPED_FILE_FAILED, null); + } + return append(record); + case SUCCESS: + return new AppendMessageRecordResult(PutMessageStatus.SUCCESS, result); + default: + return new AppendMessageRecordResult(PutMessageStatus.UNKNOWN_ERROR, result); + } + } + + @Override + public boolean clean(Long key) { + return logManager.clean(key); + } + + public void clean() { + logManager.deleteExpiredSegments(config.getMessageLogRetentionMs()); + } + + @Override + public void flush() { + logManager.flush(); + } + + LogVisitor newLogVisitor(final Long key) { + return new DelayMessageLogVisitor(logManager, key); + } + + long getMaxOffset() { + return logManager.getMaxOffset(); + } + + /** + * 4 // magic code + * 1 // attributes + * 8 // timestamp + * 8 // schedule time + * 8 // sequence + * 4 // messageId size + * ((messageIdSize > 0) ? messageIdSize : 0) + * 4 // subject size + * ((subjectSize > 0) ? subjectSize : 0) + * 8 // payload crc32 + * 4 // payload size + * ((payloadSize > 0) ? payloadSize : 0); + * + * @param subjectSize 主题大小 + * @param payloadSize 内容大小 + * @return 记录大小 + */ + private static int recordSizeWithCrc(final int messageIdSize, final int subjectSize, final int payloadSize) { + return 4 + 1 + + 8 + + 8 + + 8 + + 4 + + ((messageIdSize > 0) ? messageIdSize : 0) + + 4 + + ((subjectSize > 0) ? subjectSize : 0) + + 8 + + 4 + + ((payloadSize > 0) ? payloadSize : 0); + } + + /** + * 4 // magic code + * 1 // attributes + * 8 // timestamp + * 8 // schedule time + * 8 // sequence + * 4 // messageId size + * ((messageIdSize > 0) ? messageIdSize : 0) + * 4 // subject size + * ((subjectSize > 0) ? subjectSize : 0) + * 4 // payload size + * ((payloadSize > 0) ? payloadSize : 0); + * + * @param subjectSize 主题大小 + * @param payloadSize 内容大小 + * @return 记录大小 + */ + private static int recordSize(final int messageIdSize, final int subjectSize, final int payloadSize) { + return 4 + 1 + + 8 + + 8 + + 8 + + 4 + + ((messageIdSize > 0) ? messageIdSize : 0) + + 4 + + (subjectSize > 0 ? subjectSize : 0) + + 4 + + (payloadSize > 0 ? payloadSize : 0); + } + + long getMinOffset() { + return logManager.getMinOffset(); + } + + SegmentBuffer getMessageData(long startSyncOffset) { + LogSegment segment = logManager.locateSegment(startSyncOffset); + if (null == segment) { + return null; + } + + return segment.selectSegmentBuffer((int) (startSyncOffset % config.getMessageLogSegmentFileSize())); + } + + boolean appendData(long startOffset, ByteBuffer buffer) { + LogSegment segment = logManager.locateSegment(startOffset); + if (null == segment) { + segment = logManager.allocOrResetSegments(startOffset); + fillPreBlank(segment, startOffset); + } + return segment.appendData(buffer); + } + + private void fillPreBlank(LogSegment segment, long untilWhere) { + final ByteBuffer buffer = ByteBuffer.allocate(17); + buffer.putInt(MagicCode.MESSAGE_LOG_MAGIC_V2); + buffer.put((byte) 2); + buffer.putLong(System.currentTimeMillis()); + buffer.putInt((int) (untilWhere % config.getMessageLogSegmentFileSize())); + segment.fillPreBlank(buffer, untilWhere); + } + + private class DelayRawMessageAppender implements MessageAppender { + private final ReentrantLock lock = new ReentrantLock(); + private final ByteBuffer workingBuffer = ByteBuffer.allocate(config.getSingleMessageLimitSize()); + + @Override + public AppendMessageResult doAppend(long baseOffset, ByteBuffer targetBuffer, int freeSpace, RawMessageExtend message) { + lock.lock(); + try { + workingBuffer.clear(); + + final String messageId = message.getHeader().getMessageId(); + final byte[] messageIdBytes = messageId.getBytes(StandardCharsets.UTF_8); + final String subject = message.getHeader().getSubject(); + final byte[] subjectBytes = subject.getBytes(StandardCharsets.UTF_8); + + final long startWroteOffset = baseOffset + targetBuffer.position(); + final int recordSize = recordSizeWithCrc(messageIdBytes.length, subjectBytes.length, message.getBodySize()); + + if (recordSize > config.getSingleMessageLimitSize()) { + return new AppendMessageResult<>(AppendMessageStatus.MESSAGE_SIZE_EXCEEDED, startWroteOffset, freeSpace, null); + } + + workingBuffer.flip(); + if (recordSize != freeSpace && recordSize + MIN_RECORD_BYTES > freeSpace) { + workingBuffer.limit(freeSpace); + workingBuffer.putInt(MESSAGE_LOG_MAGIC_V1); + workingBuffer.put(MessageLogAttrEnum.ATTR_EMPTY_RECORD.getCode()); + workingBuffer.putLong(System.currentTimeMillis()); + targetBuffer.put(workingBuffer.array(), 0, freeSpace); + return new AppendMessageResult<>(AppendMessageStatus.END_OF_FILE, startWroteOffset, freeSpace, null); + } else { + workingBuffer.limit(recordSize); + workingBuffer.putInt(MESSAGE_LOG_MAGIC_V2); + workingBuffer.put(MessageLogAttrEnum.ATTR_MESSAGE_RECORD.getCode()); + workingBuffer.putLong(System.currentTimeMillis()); + workingBuffer.putLong(message.getScheduleTime()); + workingBuffer.putLong(sequence.incrementAndGet()); + workingBuffer.putInt(messageIdBytes.length); + workingBuffer.put(messageIdBytes); + workingBuffer.putInt(subjectBytes.length); + workingBuffer.put(subjectBytes); + workingBuffer.putLong(message.getHeader().getBodyCrc()); + workingBuffer.putInt(message.getBodySize()); + workingBuffer.put(message.getBody().nioBuffer()); + targetBuffer.put(workingBuffer.array(), 0, recordSize); + + final long payloadOffset = startWroteOffset + recordSize - message.getBodySize(); + return new AppendMessageResult<>(AppendMessageStatus.SUCCESS, startWroteOffset, recordSize, payloadOffset); + } + } finally { + lock.unlock(); + } + } + } + + private static class MessageLogSegmentValidator implements LogSegmentValidator { + + MessageLogSegmentValidator() { + } + + @Override + public ValidateResult validate(LogSegment segment) { + final int fileSize = segment.getFileSize(); + final ByteBuffer buffer = segment.sliceByteBuffer(); + + int position = 0; + while (true) { + if (position == fileSize) { + return new ValidateResult(ValidateStatus.COMPLETE, fileSize); + } + + final int result = consumeAndValidateMessage(buffer); + if (result == -1) { + return new ValidateResult(ValidateStatus.PARTIAL, position); + } else if (result == 0) { + return new ValidateResult(ValidateStatus.COMPLETE, fileSize); + } else { + position += result; + } + } + } + + private int consumeAndValidateMessage(final ByteBuffer buffer) { + final int magic = buffer.getInt(); + if (magic != MESSAGE_LOG_MAGIC_V1 && magic != MESSAGE_LOG_MAGIC_V2) { + return -1; + } + + final byte attributes = buffer.get(); + buffer.getLong(); + if (attributes == ATTR_SKIP_RECORD.getCode()) { + return buffer.getInt(); + } else if (attributes == ATTR_EMPTY_RECORD.getCode()) { + return 0; + } else if (attributes == ATTR_MESSAGE_RECORD.getCode()) { + return resolveRecordSize(buffer, magic); + } else { + return -1; + } + } + + private int resolveRecordSize(final ByteBuffer buffer, final int magic) { + buffer.getLong(); + buffer.getLong(); + final int messageIdSize = buffer.getInt(); + buffer.position(buffer.position() + messageIdSize); + final int subjectSize = buffer.getInt(); + buffer.position(buffer.position() + subjectSize); + if (magic >= MESSAGE_LOG_MAGIC_V2) { + buffer.getLong(); + final int payloadSize = buffer.getInt(); + final byte[] payload = new byte[payloadSize]; + buffer.get(payload); + return recordSizeWithCrc(messageIdSize, subjectSize, payloadSize); + } else { + final int payloadSize = buffer.getInt(); + buffer.position(buffer.position() + payloadSize); + return recordSize(messageIdSize, subjectSize, payloadSize); + } + } + } +} diff --git a/qmq-delay-server/src/main/java/qunar/tc/qmq/delay/store/log/ScheduleLog.java b/qmq-delay-server/src/main/java/qunar/tc/qmq/delay/store/log/ScheduleLog.java new file mode 100644 index 00000000..75ebd338 --- /dev/null +++ b/qmq-delay-server/src/main/java/qunar/tc/qmq/delay/store/log/ScheduleLog.java @@ -0,0 +1,171 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.delay.store.log; + +import com.google.common.collect.Lists; +import io.netty.buffer.ByteBuf; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import qunar.tc.qmq.common.Disposable; +import qunar.tc.qmq.delay.ScheduleIndex; +import qunar.tc.qmq.delay.base.LongHashSet; +import qunar.tc.qmq.delay.config.StoreConfiguration; +import qunar.tc.qmq.delay.store.DefaultDelaySegmentValidator; +import qunar.tc.qmq.delay.store.ScheduleLogValidatorSupport; +import qunar.tc.qmq.delay.store.appender.ScheduleSetAppender; +import qunar.tc.qmq.delay.store.model.*; +import qunar.tc.qmq.delay.store.visitor.LogVisitor; +import qunar.tc.qmq.delay.wheel.WheelLoadCursor; +import qunar.tc.qmq.protocol.producer.MessageProducerCode; + +import java.io.File; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.function.Consumer; + +import static qunar.tc.qmq.delay.store.ScheduleLogValidatorSupport.getSupport; + + +/** + * @author xufeng.deng dennisdxf@gmail.com + * @since 2018-08-02 17:47 + */ +public class ScheduleLog implements Log, Disposable { + private static final Logger LOGGER = LoggerFactory.getLogger(ScheduleLog.class); + + private final ScheduleSet scheduleSet; + private final AtomicBoolean open; + private final StoreConfiguration config; + + public ScheduleLog(StoreConfiguration storeConfiguration) { + final ScheduleSetSegmentContainer setContainer = new ScheduleSetSegmentContainer( + storeConfiguration, new File(storeConfiguration.getScheduleLogStorePath()) + , new DefaultDelaySegmentValidator(), new ScheduleSetAppender(storeConfiguration.getSingleMessageLimitSize())); + + this.config = storeConfiguration; + this.scheduleSet = new ScheduleSet(setContainer); + this.open = new AtomicBoolean(true); + reValidate(storeConfiguration.getSingleMessageLimitSize()); + } + + private void reValidate(int singleMessageLimitSize) { + ScheduleLogValidatorSupport support = ScheduleLogValidatorSupport.getSupport(config); + Map offsets = support.loadScheduleOffsetCheckpoint(); + scheduleSet.reValidate(offsets, singleMessageLimitSize); + } + + @Override + public AppendLogResult append(LogRecord record) { + if (!open.get()) { + return new AppendLogResult<>(MessageProducerCode.STORE_ERROR, "schedule log closed"); + } + AppendLogResult> result = scheduleSet.append(record); + int code = result.getCode(); + if (MessageProducerCode.SUCCESS != code) { + LOGGER.error("appendMessageLog schedule set error,log:{} {},code:{}", record.getSubject(), record.getMessageId(), code); + return new AppendLogResult<>(MessageProducerCode.STORE_ERROR, "appendScheduleSetError"); + } + + RecordResult recordResult = result.getAdditional(); + ByteBuf index = buildIndexRecord(record.getScheduleTime(), recordResult.getResult().getWroteOffset(), + recordResult.getResult().getWroteBytes(), recordResult.getResult().getAdditional().getSequence()); + + return new AppendLogResult<>(MessageProducerCode.SUCCESS, "", index); + } + + private ByteBuf buildIndexRecord(long scheduleTime, long offset, int size, long sequence) { + return ScheduleIndex.buildIndex(scheduleTime, offset, size, sequence); + } + + @Override + public boolean clean(Long key) { + return scheduleSet.clean(key); + } + + @Override + public void flush() { + if (open.get()) { + scheduleSet.flush(); + } + } + + public List recoverLogRecord(List pureRecords) { + try { + List records = Lists.newArrayListWithCapacity(pureRecords.size()); + for (ByteBuf record : pureRecords) { + ScheduleSetRecord logRecord = scheduleSet.recoverRecord(record); + if (logRecord == null) { + LOGGER.error("schedule log recover null record"); + continue; + } + + records.add(logRecord); + } + + return records; + } finally { + ScheduleIndex.release(pureRecords); + } + } + + public void clean() { + scheduleSet.clean(); + } + + public WheelLoadCursor.Cursor loadUnDispatch(ScheduleSetSegment segment, final LongHashSet dispatchedSet, final Consumer func) { + LogVisitor visitor = segment.newVisitor(0, config.getSingleMessageLimitSize()); + try { + long offset = 0; + while (true) { + Optional recordOptional = visitor.nextRecord(); + if (!recordOptional.isPresent()) break; + ByteBuf index = recordOptional.get(); + long sequence = ScheduleIndex.sequence(index); + offset = ScheduleIndex.offset(index) + ScheduleIndex.size(index); + if (!dispatchedSet.contains(sequence)) { + func.accept(index); + } else { + ScheduleIndex.release(index); + } + } + return new WheelLoadCursor.Cursor(segment.getSegmentBaseOffset(), offset); + } finally { + visitor.close(); + LOGGER.info("schedule log recover {} which is need to continue to dispatch.", segment.getSegmentBaseOffset()); + } + } + + public ScheduleSetSegment loadSegment(int segmentBaseOffset) { + return scheduleSet.loadSegment(segmentBaseOffset); + } + + @Override + public void destroy() { + open.set(false); + getSupport(config).saveScheduleOffsetCheckpoint(checkOffsets()); + } + + private Map checkOffsets() { + return scheduleSet.countSegments(); + } + + public int higherBaseOffset(int low) { + return scheduleSet.higherBaseOffset(low); + } +} diff --git a/qmq-delay-server/src/main/java/qunar/tc/qmq/delay/store/log/ScheduleOffsetResolver.java b/qmq-delay-server/src/main/java/qunar/tc/qmq/delay/store/log/ScheduleOffsetResolver.java new file mode 100644 index 00000000..8479bf2a --- /dev/null +++ b/qmq-delay-server/src/main/java/qunar/tc/qmq/delay/store/log/ScheduleOffsetResolver.java @@ -0,0 +1,39 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.delay.store.log; + +import org.joda.time.LocalDateTime; + + +/** + * @author xufeng.deng dennisdxf@gmail.com + * @since 2018-07-13 11:16 + */ +public class ScheduleOffsetResolver { + + static { + LocalDateTime.now(); + } + + public static int resolveSegment(long offset) { + LocalDateTime localDateTime = new LocalDateTime(offset); + return localDateTime.getYear() * 1000000 + + localDateTime.getMonthOfYear() * 10000 + + localDateTime.getDayOfMonth() * 100 + + localDateTime.getHourOfDay(); + } +} diff --git a/qmq-delay-server/src/main/java/qunar/tc/qmq/delay/store/log/ScheduleSet.java b/qmq-delay-server/src/main/java/qunar/tc/qmq/delay/store/log/ScheduleSet.java new file mode 100644 index 00000000..fb78793f --- /dev/null +++ b/qmq-delay-server/src/main/java/qunar/tc/qmq/delay/store/log/ScheduleSet.java @@ -0,0 +1,64 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.delay.store.log; + +import io.netty.buffer.ByteBuf; +import qunar.tc.qmq.delay.ScheduleIndex; +import qunar.tc.qmq.delay.store.model.LogRecord; +import qunar.tc.qmq.delay.store.model.RecordResult; +import qunar.tc.qmq.delay.store.model.ScheduleSetRecord; +import qunar.tc.qmq.delay.store.model.ScheduleSetSequence; + +import java.util.Map; + +/** + * @author xufeng.deng dennisdxf@gmail.com + * @since 2018-07-19 13:50 + */ +public class ScheduleSet extends AbstractDelayLog { + + public ScheduleSet(SegmentContainer, LogRecord> container) { + super(container); + } + + ScheduleSetRecord recoverRecord(ByteBuf index) { + long scheduleTime = ScheduleIndex.scheduleTime(index); + long offset = ScheduleIndex.offset(index); + int size = ScheduleIndex.size(index); + return ((ScheduleSetSegmentContainer) container).recover(scheduleTime, size, offset); + } + + public void clean() { + ((ScheduleSetSegmentContainer) container).clean(); + } + + public ScheduleSetSegment loadSegment(int segmentBaseOffset) { + return ((ScheduleSetSegmentContainer) container).loadSegment(segmentBaseOffset); + } + + synchronized Map countSegments() { + return ((ScheduleSetSegmentContainer) container).countSegments(); + } + + void reValidate(final Map offsets, int singleMessageLimitSize) { + ((ScheduleSetSegmentContainer) container).reValidate(offsets, singleMessageLimitSize); + } + + int higherBaseOffset(int low) { + return ((ScheduleSetSegmentContainer) container).higherBaseOffset(low); + } +} diff --git a/qmq-delay-server/src/main/java/qunar/tc/qmq/delay/store/log/ScheduleSetSegment.java b/qmq-delay-server/src/main/java/qunar/tc/qmq/delay/store/log/ScheduleSetSegment.java new file mode 100644 index 00000000..f77c71a7 --- /dev/null +++ b/qmq-delay-server/src/main/java/qunar/tc/qmq/delay/store/log/ScheduleSetSegment.java @@ -0,0 +1,106 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.delay.store.log; + +import io.netty.buffer.ByteBuf; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import qunar.tc.qmq.delay.ScheduleIndex; +import qunar.tc.qmq.delay.store.model.ScheduleSetRecord; +import qunar.tc.qmq.delay.store.model.ScheduleSetSequence; +import qunar.tc.qmq.delay.store.visitor.LogVisitor; +import qunar.tc.qmq.delay.store.visitor.ScheduleIndexVisitor; + +import java.io.File; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; +import java.util.Optional; + +/** + * @author xufeng.deng dennisdxf@gmail.com + * @since 2018-07-19 12:23 + */ +public class ScheduleSetSegment extends AbstractDelaySegment { + private static final Logger LOGGER = LoggerFactory.getLogger(ScheduleSetSegment.class); + + ScheduleSetSegment(File file) throws IOException { + super(file); + } + + @Override + public long validate() throws IOException { + return fileChannel.size(); + } + + ScheduleSetRecord recover(long offset, int size) { + // 交给gc,不能给每个segment分配一个局部buffer + ByteBuffer recoverBuffer = ByteBuffer.allocate(size); + try { + int bytes = fileChannel.read(recoverBuffer, offset); + if (bytes != size) { + LOGGER.error("schedule set segment recovered failed,need read more bytes,segment:{},offset:{},size:{}, readBytes:{}, segmentTotalSize:{}", fileName, offset, size, bytes, fileChannel.size()); + return null; + } + recoverBuffer.flip(); + long scheduleTime = recoverBuffer.getLong(); + long sequence = recoverBuffer.getLong(); + recoverBuffer.getInt(); + + int messageIdSize = recoverBuffer.getInt(); + byte[] messageId = new byte[messageIdSize]; + recoverBuffer.get(messageId); + int subjectSize = recoverBuffer.getInt(); + byte[] subject = new byte[subjectSize]; + recoverBuffer.get(subject); + return new ScheduleSetRecord(new String(messageId, StandardCharsets.UTF_8), new String(subject, StandardCharsets.UTF_8), scheduleTime, offset, size, sequence, recoverBuffer.slice()); + } catch (Throwable e) { + LOGGER.error("schedule set segment recovered error,segment:{}, offset-size:{} {}", fileName, offset, size, e); + return null; + } + } + + void loadOffset(long scheduleSetWroteOffset) { + if (getWrotePosition() != scheduleSetWroteOffset) { + setWrotePosition(scheduleSetWroteOffset); + setFlushedPosition(scheduleSetWroteOffset); + LOGGER.warn("schedule set load offset,exist invalid message,segment base offset:{}, wroteOffset:{}", getSegmentBaseOffset(), scheduleSetWroteOffset); + } + } + + public LogVisitor newVisitor(long from, int singleMessageLimitSize) { + return new ScheduleIndexVisitor(from, fileChannel, singleMessageLimitSize); + } + + long doValidate(int singleMessageLimitSize) { + LOGGER.info("validate schedule log {}", getSegmentBaseOffset()); + LogVisitor visitor = newVisitor(0, singleMessageLimitSize); + try { + while (true) { + Optional optionalRecord = visitor.nextRecord(); + if (optionalRecord.isPresent()) { + ScheduleIndex.release(optionalRecord.get()); + continue; + } + break; + } + return visitor.visitedBufferSize(); + } finally { + visitor.close(); + } + } +} diff --git a/qmq-delay-server/src/main/java/qunar/tc/qmq/delay/store/log/ScheduleSetSegmentContainer.java b/qmq-delay-server/src/main/java/qunar/tc/qmq/delay/store/log/ScheduleSetSegmentContainer.java new file mode 100644 index 00000000..7c9e8d3f --- /dev/null +++ b/qmq-delay-server/src/main/java/qunar/tc/qmq/delay/store/log/ScheduleSetSegmentContainer.java @@ -0,0 +1,147 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.delay.store.log; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import qunar.tc.qmq.delay.config.StoreConfiguration; +import qunar.tc.qmq.delay.store.DelaySegmentValidator; +import qunar.tc.qmq.delay.store.appender.LogAppender; +import qunar.tc.qmq.delay.store.model.*; +import qunar.tc.qmq.store.AppendMessageResult; +import qunar.tc.qmq.store.PutMessageStatus; + +import java.io.File; +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; + +import static qunar.tc.qmq.delay.store.log.ScheduleOffsetResolver.resolveSegment; + +/** + * @author xufeng.deng dennisdxf@gmail.com + * @since 2018-07-19 13:44 + */ +public class ScheduleSetSegmentContainer extends AbstractDelaySegmentContainer { + private static final Logger LOGGER = LoggerFactory.getLogger(ScheduleSetSegmentContainer.class); + + private final StoreConfiguration config; + + public ScheduleSetSegmentContainer(StoreConfiguration config, File logDir, DelaySegmentValidator validator, LogAppender appender) { + super(logDir, validator, appender); + this.config = config; + } + + @Override + protected void loadLogs(DelaySegmentValidator validator) { + LOGGER.info("Loading logs."); + File[] files = this.logDir.listFiles(); + if (files != null) { + for (final File file : files) { + if (file.getName().startsWith(".")) { + continue; + } + if (file.isDirectory()) { + continue; + } + + DelaySegment segment; + try { + segment = new ScheduleSetSegment(file); + long size = validator.validate(segment); + segment.setWrotePosition(size); + segment.setFlushedPosition(size); + segments.put(segment.getSegmentBaseOffset(), segment); + } catch (IOException e) { + LOGGER.error("Load {} failed.", file.getAbsolutePath(), e); + } + } + } + LOGGER.info("Load logs done."); + } + + @Override + protected RecordResult retResult(AppendMessageResult result) { + switch (result.getStatus()) { + case SUCCESS: + return new AppendScheduleLogRecordResult(PutMessageStatus.SUCCESS, result); + default: + return new AppendScheduleLogRecordResult(PutMessageStatus.UNKNOWN_ERROR, result); + } + } + + @Override + protected DelaySegment allocSegment(int segmentBaseOffset) { + File nextSegmentFile = new File(logDir, String.valueOf(segmentBaseOffset)); + try { + DelaySegment logSegment = new ScheduleSetSegment(nextSegmentFile); + segments.put(segmentBaseOffset, logSegment); + LOGGER.info("alloc new schedule set segment file {}", ((ScheduleSetSegment) logSegment).fileName); + return logSegment; + } catch (IOException e) { + LOGGER.error("Failed create new schedule set segment file. file: {}", nextSegmentFile.getAbsolutePath(), e); + } + return null; + } + + ScheduleSetRecord recover(long scheduleTime, int size, long offset) { + ScheduleSetSegment segment = (ScheduleSetSegment) locateSegment(scheduleTime); + if (null == segment) { + LOGGER.error("schedule set recover null value, scheduleTime:{}, size:{}, offset:{}", scheduleTime, size, offset); + return null; + } + + return segment.recover(offset, size); + } + + public void clean() { + Integer checkTime = resolveSegment(System.currentTimeMillis() - config.getDispatchLogKeepTime() - config.getCheckCleanTimeBeforeDispatch()); + for (DelaySegment segment : segments.values()) { + if (segment.getSegmentBaseOffset() < checkTime) { + clean((long) segment.getSegmentBaseOffset()); + } + } + } + + ScheduleSetSegment loadSegment(int segmentBaseOffset) { + return (ScheduleSetSegment) segments.get(segmentBaseOffset); + } + + Map countSegments() { + final Map offsets = new HashMap<>(segments.size()); + segments.values().forEach(segment -> offsets.put(segment.getSegmentBaseOffset(), segment.getWrotePosition())); + return offsets; + } + + void reValidate(final Map offsets, int singleMessageLimitSize) { + segments.values().parallelStream().forEach(segment -> { + Long offset = offsets.get(segment.getSegmentBaseOffset()); + long wrotePosition = segment.getWrotePosition(); + if (null == offset || offset != wrotePosition) { + offset = doValidate((ScheduleSetSegment) segment, singleMessageLimitSize); + } else { + offset = wrotePosition; + } + + ((ScheduleSetSegment) segment).loadOffset(offset); + }); + } + + private long doValidate(ScheduleSetSegment segment, int singleMessageLimitSize) { + return segment.doValidate(singleMessageLimitSize); + } +} diff --git a/qmq-delay-server/src/main/java/qunar/tc/qmq/delay/store/log/SegmentContainer.java b/qmq-delay-server/src/main/java/qunar/tc/qmq/delay/store/log/SegmentContainer.java new file mode 100644 index 00000000..42463ffe --- /dev/null +++ b/qmq-delay-server/src/main/java/qunar/tc/qmq/delay/store/log/SegmentContainer.java @@ -0,0 +1,31 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.delay.store.log; + + +/** + * @author xufeng.deng dennisdxf@gmail.com + * @since 2018-07-19 10:03 + */ +public interface SegmentContainer { + + R append(T record); + + boolean clean(Long key); + + void flush(); +} diff --git a/qmq-delay-server/src/main/java/qunar/tc/qmq/delay/store/model/AppendDispatchRecordResult.java b/qmq-delay-server/src/main/java/qunar/tc/qmq/delay/store/model/AppendDispatchRecordResult.java new file mode 100644 index 00000000..e4c71943 --- /dev/null +++ b/qmq-delay-server/src/main/java/qunar/tc/qmq/delay/store/model/AppendDispatchRecordResult.java @@ -0,0 +1,45 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.delay.store.model; + +import qunar.tc.qmq.store.AppendMessageResult; +import qunar.tc.qmq.store.PutMessageStatus; + +/** + * @author xufeng.deng dennisdxf@gmail.com + * @since 2018-07-20 14:55 + */ +public class AppendDispatchRecordResult implements RecordResult { + private PutMessageStatus status; + private AppendMessageResult result; + + public AppendDispatchRecordResult(PutMessageStatus status, AppendMessageResult result) { + this.status = status; + this.result = result; + } + + @Override + public PutMessageStatus getStatus() { + return status; + } + + @Override + public AppendMessageResult getResult() { + return result; + } + +} diff --git a/qmq-delay-server/src/main/java/qunar/tc/qmq/delay/store/model/AppendLogResult.java b/qmq-delay-server/src/main/java/qunar/tc/qmq/delay/store/model/AppendLogResult.java new file mode 100644 index 00000000..6db3b740 --- /dev/null +++ b/qmq-delay-server/src/main/java/qunar/tc/qmq/delay/store/model/AppendLogResult.java @@ -0,0 +1,50 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.delay.store.model; + +/** + * @author xufeng.deng dennisdxf@gmail.com + * @since 2018-07-12 15:25 + */ +public class AppendLogResult { + private int code; + private String remark; + private T additional; + + public AppendLogResult(int code, String remark) { + this(code, remark, null); + } + + public AppendLogResult(int code, String remark, T additional) { + this.code = code; + this.remark = remark; + this.additional = additional; + } + + public int getCode() { + return code; + } + + public String getRemark() { + return remark; + } + + public T getAdditional() { + return additional; + } + +} diff --git a/qmq-delay-server/src/main/java/qunar/tc/qmq/delay/store/model/AppendMessageRecordResult.java b/qmq-delay-server/src/main/java/qunar/tc/qmq/delay/store/model/AppendMessageRecordResult.java new file mode 100644 index 00000000..6b7321ca --- /dev/null +++ b/qmq-delay-server/src/main/java/qunar/tc/qmq/delay/store/model/AppendMessageRecordResult.java @@ -0,0 +1,45 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.delay.store.model; + +import qunar.tc.qmq.store.AppendMessageResult; +import qunar.tc.qmq.store.PutMessageStatus; + +/** + * @author xufeng.deng dennisdxf@gmail.com + * @since 2018-07-19 9:48 + */ +public class AppendMessageRecordResult implements RecordResult { + private PutMessageStatus status; + private AppendMessageResult result; + + public AppendMessageRecordResult(PutMessageStatus status, AppendMessageResult result) { + this.status = status; + this.result = result; + } + + @Override + public PutMessageStatus getStatus() { + return status; + } + + @Override + public AppendMessageResult getResult() { + return result; + } + +} diff --git a/qmq-delay-server/src/main/java/qunar/tc/qmq/delay/store/model/AppendRecordResult.java b/qmq-delay-server/src/main/java/qunar/tc/qmq/delay/store/model/AppendRecordResult.java new file mode 100644 index 00000000..a9127a2d --- /dev/null +++ b/qmq-delay-server/src/main/java/qunar/tc/qmq/delay/store/model/AppendRecordResult.java @@ -0,0 +1,62 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.delay.store.model; + +import qunar.tc.qmq.store.AppendMessageStatus; + +import java.nio.ByteBuffer; + +/** + * @author xufeng.deng dennisdxf@gmail.com + * @since 2018-07-19 12:01 + */ +public class AppendRecordResult { + private final AppendMessageStatus status; + private final long wroteOffset; + private final int wroteBytes; + private final ByteBuffer buffer; + private T additional; + + public AppendRecordResult(AppendMessageStatus status, long wroteOffset, int wroteBytes, ByteBuffer buffer) { + this(status, wroteOffset, wroteBytes, buffer, null); + } + + public AppendRecordResult(AppendMessageStatus status, long wroteOffset, int wroteBytes, ByteBuffer buffer, T additional) { + this.status = status; + this.wroteOffset = wroteOffset; + this.wroteBytes = wroteBytes; + this.buffer = buffer; + this.additional = additional; + } + + public AppendMessageStatus getStatus() { + return status; + } + + public int getWroteBytes() { + return wroteBytes; + } + + public T getAdditional() { + return additional; + } + + public ByteBuffer getBuffer() { + return buffer; + } + +} diff --git a/qmq-delay-server/src/main/java/qunar/tc/qmq/delay/store/model/AppendScheduleLogRecordResult.java b/qmq-delay-server/src/main/java/qunar/tc/qmq/delay/store/model/AppendScheduleLogRecordResult.java new file mode 100644 index 00000000..00be50d5 --- /dev/null +++ b/qmq-delay-server/src/main/java/qunar/tc/qmq/delay/store/model/AppendScheduleLogRecordResult.java @@ -0,0 +1,47 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.delay.store.model; + +import qunar.tc.qmq.store.AppendMessageResult; +import qunar.tc.qmq.store.PutMessageStatus; + +/** + * @author xufeng.deng dennisdxf@gmail.com + * @since 2018-07-19 15:01 + */ +public class AppendScheduleLogRecordResult implements RecordResult { + private PutMessageStatus status; + + private AppendMessageResult result; + + + public AppendScheduleLogRecordResult(PutMessageStatus status, AppendMessageResult result) { + this.status = status; + this.result = result; + } + + @Override + public PutMessageStatus getStatus() { + return status; + } + + @Override + public AppendMessageResult getResult() { + return result; + } + +} diff --git a/qmq-delay-server/src/main/java/qunar/tc/qmq/delay/store/model/DispatchLogRecord.java b/qmq-delay-server/src/main/java/qunar/tc/qmq/delay/store/model/DispatchLogRecord.java new file mode 100644 index 00000000..b381240d --- /dev/null +++ b/qmq-delay-server/src/main/java/qunar/tc/qmq/delay/store/model/DispatchLogRecord.java @@ -0,0 +1,71 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.delay.store.model; + +import java.nio.ByteBuffer; + +/** + * @author xufeng.deng dennisdxf@gmail.com + * @since 2018-07-20 14:44 + */ +public class DispatchLogRecord implements LogRecord { + private final LogRecordHeader header; + + public DispatchLogRecord(String subject, String messageId, long scheduleTime, long sequence) { + this.header = new LogRecordHeader(subject, messageId, scheduleTime, sequence); + } + + @Override + public String getSubject() { + return header.getSubject(); + } + + @Override + public String getMessageId() { + return header.getMessageId(); + } + + @Override + public long getScheduleTime() { + return header.getScheduleTime(); + } + + @Override + public int getPayloadSize() { + return 0; + } + + @Override + public ByteBuffer getRecord() { + return null; + } + + @Override + public long getStartWroteOffset() { + return 0; + } + + @Override + public int getRecordSize() { + return Long.BYTES; + } + + @Override + public long getSequence() { + return header.getSequence(); + } +} diff --git a/qmq-delay-server/src/main/java/qunar/tc/qmq/delay/store/model/LogRecord.java b/qmq-delay-server/src/main/java/qunar/tc/qmq/delay/store/model/LogRecord.java new file mode 100644 index 00000000..bdd4ccc0 --- /dev/null +++ b/qmq-delay-server/src/main/java/qunar/tc/qmq/delay/store/model/LogRecord.java @@ -0,0 +1,42 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.delay.store.model; + +import java.nio.ByteBuffer; + +/** + * @author xufeng.deng dennisdxf@gmail.com + * @since 2018-07-16 10:55 + */ +public interface LogRecord { + + String getSubject(); + + String getMessageId(); + + long getScheduleTime(); + + int getPayloadSize(); + + ByteBuffer getRecord(); + + long getStartWroteOffset(); + + int getRecordSize(); + + long getSequence(); +} diff --git a/qmq-delay-server/src/main/java/qunar/tc/qmq/delay/store/model/LogRecordHeader.java b/qmq-delay-server/src/main/java/qunar/tc/qmq/delay/store/model/LogRecordHeader.java new file mode 100644 index 00000000..97c6cbea --- /dev/null +++ b/qmq-delay-server/src/main/java/qunar/tc/qmq/delay/store/model/LogRecordHeader.java @@ -0,0 +1,63 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.delay.store.model; + +import qunar.tc.qmq.base.MessageHeader; + + +/** + * @author xufeng.deng dennisdxf@gmail.com + * @since 2018-07-14 11:14 + */ +public class LogRecordHeader { + private MessageHeader header; + private long scheduleTime; + private long sequence; + + public LogRecordHeader(String subject, String messageId, long scheduleTime, long sequence) { + this.header = new MessageHeader(); + this.header.setSubject(subject); + this.header.setMessageId(messageId); + this.scheduleTime = scheduleTime; + this.sequence = sequence; + } + + public String getSubject() { + return header.getSubject(); + } + + public String getMessageId() { + return header.getMessageId(); + } + + public long getScheduleTime() { + return scheduleTime; + } + + public long getSequence() { + return sequence; + } + + @Override + public String toString() { + return "LogRecordHeader{" + + "header=" + header + + ", scheduleTime=" + scheduleTime + + ", sequence=" + sequence + + '}'; + } +} diff --git a/qmq-delay-server/src/main/java/qunar/tc/qmq/delay/store/model/MessageLogAttrEnum.java b/qmq-delay-server/src/main/java/qunar/tc/qmq/delay/store/model/MessageLogAttrEnum.java new file mode 100644 index 00000000..7747e603 --- /dev/null +++ b/qmq-delay-server/src/main/java/qunar/tc/qmq/delay/store/model/MessageLogAttrEnum.java @@ -0,0 +1,48 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.delay.store.model; + +/** + * @author xufeng.deng dennisdxf@gmail.com + * @since 2018-07-11 21:21 + */ +public enum MessageLogAttrEnum { + /** + * 正常记录 + */ + ATTR_MESSAGE_RECORD((byte) 0), + /** + * 空记录 + */ + ATTR_EMPTY_RECORD((byte) 1), + /** + * 需要跳过记录 + */ + ATTR_SKIP_RECORD((byte) 2); + + private byte code; + + MessageLogAttrEnum(byte code) { + this.code = code; + } + + public byte getCode() { + return code; + } + + +} diff --git a/qmq-delay-server/src/main/java/qunar/tc/qmq/delay/store/model/MessageLogRecord.java b/qmq-delay-server/src/main/java/qunar/tc/qmq/delay/store/model/MessageLogRecord.java new file mode 100644 index 00000000..24701a17 --- /dev/null +++ b/qmq-delay-server/src/main/java/qunar/tc/qmq/delay/store/model/MessageLogRecord.java @@ -0,0 +1,81 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.delay.store.model; + + +import java.nio.ByteBuffer; + +/** + * @author xufeng.deng dennisdxf@gmail.com + * @since 2018-07-12 10:20 + */ +public class MessageLogRecord implements LogRecord { + private LogRecordHeader header; + private int wroteBytes; + private long startWroteOffset; + private int payloadSize; + private ByteBuffer buffer; + + public MessageLogRecord(LogRecordHeader header, int wroteBytes, long startWroteOffset, int payloadSize, ByteBuffer buffer) { + this.header = header; + this.wroteBytes = wroteBytes; + this.startWroteOffset = startWroteOffset; + this.payloadSize = payloadSize; + this.buffer = buffer; + } + + @Override + public long getStartWroteOffset() { + return startWroteOffset; + } + + @Override + public int getRecordSize() { + return wroteBytes; + } + + @Override + public int getPayloadSize() { + return payloadSize; + } + + @Override + public ByteBuffer getRecord() { + return buffer; + } + + @Override + public String getSubject() { + return header.getSubject(); + } + + @Override + public String getMessageId() { + return header.getMessageId(); + } + + @Override + public long getScheduleTime() { + return header.getScheduleTime(); + } + + @Override + public long getSequence() { + return header.getSequence(); + } + +} diff --git a/qmq-delay-server/src/main/java/qunar/tc/qmq/delay/store/model/NopeRecordResult.java b/qmq-delay-server/src/main/java/qunar/tc/qmq/delay/store/model/NopeRecordResult.java new file mode 100644 index 00000000..25ef50bb --- /dev/null +++ b/qmq-delay-server/src/main/java/qunar/tc/qmq/delay/store/model/NopeRecordResult.java @@ -0,0 +1,45 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.delay.store.model; + +import qunar.tc.qmq.store.AppendMessageResult; +import qunar.tc.qmq.store.AppendMessageStatus; +import qunar.tc.qmq.store.PutMessageStatus; + + +/** + * @author xufeng.deng dennisdxf@gmail.com + * @since 2018-07-19 13:29 + */ +public class NopeRecordResult implements RecordResult { + + private PutMessageStatus status; + + public NopeRecordResult(PutMessageStatus status) { + this.status = status; + } + + @Override + public PutMessageStatus getStatus() { + return status; + } + + @Override + public AppendMessageResult getResult() { + return new AppendMessageResult<>(AppendMessageStatus.UNKNOWN_ERROR, -1, -1); + } +} diff --git a/qmq-delay-server/src/main/java/qunar/tc/qmq/delay/store/model/RawMessageExtend.java b/qmq-delay-server/src/main/java/qunar/tc/qmq/delay/store/model/RawMessageExtend.java new file mode 100644 index 00000000..a540ec65 --- /dev/null +++ b/qmq-delay-server/src/main/java/qunar/tc/qmq/delay/store/model/RawMessageExtend.java @@ -0,0 +1,49 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.delay.store.model; + +import io.netty.buffer.ByteBuf; +import qunar.tc.qmq.base.MessageHeader; +import qunar.tc.qmq.base.RawMessage; + +/** + * @author xufeng.deng dennisdxf@gmail.com + * @since 2018-07-12 14:35 + */ +public class RawMessageExtend extends RawMessage { + private long scheduleTime; + + public RawMessageExtend(MessageHeader header, ByteBuf body, int size, long scheduleTime) { + super(header, body, size); + this.scheduleTime = scheduleTime; + } + + public long getScheduleTime() { + return scheduleTime; + } + + public void setScheduleTime(long scheduleTime) { + this.scheduleTime = scheduleTime; + } + + @Override + public String toString() { + return "RawMessageExtend{" + + "scheduleTime=" + scheduleTime + + '}'; + } +} diff --git a/qmq-delay-server/src/main/java/qunar/tc/qmq/delay/store/model/RecordResult.java b/qmq-delay-server/src/main/java/qunar/tc/qmq/delay/store/model/RecordResult.java new file mode 100644 index 00000000..b763dbc9 --- /dev/null +++ b/qmq-delay-server/src/main/java/qunar/tc/qmq/delay/store/model/RecordResult.java @@ -0,0 +1,31 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.delay.store.model; + +import qunar.tc.qmq.store.AppendMessageResult; +import qunar.tc.qmq.store.PutMessageStatus; + +/** + * @author xufeng.deng dennisdxf@gmail.com + * @since 2018-07-19 10:44 + */ +public interface RecordResult { + + PutMessageStatus getStatus(); + + AppendMessageResult getResult(); +} diff --git a/qmq-delay-server/src/main/java/qunar/tc/qmq/delay/store/model/ScheduleSetRecord.java b/qmq-delay-server/src/main/java/qunar/tc/qmq/delay/store/model/ScheduleSetRecord.java new file mode 100644 index 00000000..a13d5426 --- /dev/null +++ b/qmq-delay-server/src/main/java/qunar/tc/qmq/delay/store/model/ScheduleSetRecord.java @@ -0,0 +1,88 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.delay.store.model; + + +import java.nio.ByteBuffer; + +/** + * @author xufeng.deng dennisdxf@gmail.com + * @since 2018-07-13 11:00 + */ +public class ScheduleSetRecord implements LogRecord { + private final LogRecordHeader header; + private final long startOffset; + private final int recordSize; + + private ByteBuffer record; + + public ScheduleSetRecord(String messageId, String subject, long scheduleTime, long startOffset, int recordSize, long sequence, ByteBuffer record) { + this.header = new LogRecordHeader(subject, messageId, scheduleTime, sequence); + this.startOffset = startOffset; + this.recordSize = recordSize; + this.record = record; + } + + @Override + public String getSubject() { + return header.getSubject(); + } + + @Override + public String getMessageId() { + return header.getMessageId(); + } + + @Override + public long getScheduleTime() { + return header.getScheduleTime(); + } + + @Override + public int getPayloadSize() { + return -1; + } + + @Override + public ByteBuffer getRecord() { + return record; + } + + @Override + public long getStartWroteOffset() { + return startOffset; + } + + @Override + public int getRecordSize() { + return recordSize; + } + + @Override + public long getSequence() { + return header.getSequence(); + } + + @Override + public String toString() { + return "ScheduleSetRecord{" + + "header=" + header + + ", recordSize=" + recordSize + + ", record=" + record + + '}'; + } +} diff --git a/qmq-delay-server/src/main/java/qunar/tc/qmq/delay/store/model/ScheduleSetSequence.java b/qmq-delay-server/src/main/java/qunar/tc/qmq/delay/store/model/ScheduleSetSequence.java new file mode 100644 index 00000000..5fafaf71 --- /dev/null +++ b/qmq-delay-server/src/main/java/qunar/tc/qmq/delay/store/model/ScheduleSetSequence.java @@ -0,0 +1,39 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.delay.store.model; + +/** + * @author xufeng.deng dennisdxf@gmail.com + * @since 2018-07-23 17:31 + */ +public class ScheduleSetSequence { + private final long scheduleTime; + private final long sequence; + + public ScheduleSetSequence(long scheduleTime, long sequence) { + this.scheduleTime = scheduleTime; + this.sequence = sequence; + } + + public long getScheduleTime() { + return scheduleTime; + } + + public long getSequence() { + return sequence; + } +} diff --git a/qmq-delay-server/src/main/java/qunar/tc/qmq/delay/store/visitor/AbstractLogVisitor.java b/qmq-delay-server/src/main/java/qunar/tc/qmq/delay/store/visitor/AbstractLogVisitor.java new file mode 100644 index 00000000..a1c682c6 --- /dev/null +++ b/qmq-delay-server/src/main/java/qunar/tc/qmq/delay/store/visitor/AbstractLogVisitor.java @@ -0,0 +1,97 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.delay.store.visitor; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import sun.misc.Cleaner; +import sun.nio.ch.DirectBuffer; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.channels.FileChannel; +import java.util.Optional; +import java.util.concurrent.atomic.AtomicLong; + +/** + * Created by zhaohui.yu + * 8/20/18 + */ +public abstract class AbstractLogVisitor implements LogVisitor { + private static final Logger LOGGER = LoggerFactory.getLogger(DispatchLogVisitor.class); + + private final FileChannel fileChannel; + private final AtomicLong visited; + private final AtomicLong visitedSnapshot; + private final ByteBuffer buffer; + + AbstractLogVisitor(long from, FileChannel fileChannel, int workingSize) { + this.fileChannel = fileChannel; + this.visited = new AtomicLong(from); + this.visitedSnapshot = new AtomicLong(from); + buffer = ByteBuffer.allocateDirect(workingSize); + try { + fileChannel.read(buffer, visited.get()); + buffer.flip(); + } catch (IOException e) { + LOGGER.error("load dispatch log visitor error", e); + } + } + + @Override + public Optional nextRecord() { + int start = buffer.position(); + Optional optional = readOneRecord(buffer); + int delta = buffer.position() - start; + + if (optional.isPresent()) visited.addAndGet(delta); + else if (visited.get() > visitedSnapshot.get()) { + if (reAlloc()) return nextRecord(); + } + return optional; + } + + private boolean reAlloc() { + try { + buffer.clear(); + int bytes = fileChannel.read(buffer, visited.get()); + if (bytes > 0) { + buffer.flip(); + visitedSnapshot.addAndGet(visited.get() - visitedSnapshot.get()); + return true; + } + } catch (IOException e) { + LOGGER.error("load visitor nextRecord error", e); + } + + return false; + } + + protected abstract Optional readOneRecord(ByteBuffer buffer); + + @Override + public long visitedBufferSize() { + return visited.get(); + } + + @Override + public void close() { + if (buffer == null) return; + Cleaner cleaner = ((DirectBuffer) buffer).cleaner(); + cleaner.clean(); + } +} diff --git a/qmq-delay-server/src/main/java/qunar/tc/qmq/delay/store/visitor/DelayMessageLogVisitor.java b/qmq-delay-server/src/main/java/qunar/tc/qmq/delay/store/visitor/DelayMessageLogVisitor.java new file mode 100644 index 00000000..300f5c96 --- /dev/null +++ b/qmq-delay-server/src/main/java/qunar/tc/qmq/delay/store/visitor/DelayMessageLogVisitor.java @@ -0,0 +1,232 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.delay.store.visitor; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import qunar.tc.qmq.delay.store.model.LogRecord; +import qunar.tc.qmq.delay.store.model.LogRecordHeader; +import qunar.tc.qmq.delay.store.model.MessageLogRecord; +import qunar.tc.qmq.store.*; + +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; +import java.util.Optional; +import java.util.concurrent.atomic.AtomicInteger; + +import static qunar.tc.qmq.delay.store.model.MessageLogAttrEnum.ATTR_EMPTY_RECORD; +import static qunar.tc.qmq.delay.store.model.MessageLogAttrEnum.ATTR_SKIP_RECORD; + +/** + * @author xufeng.deng dennisdxf@gmail.com + * @since 2018-07-11 18:33 + */ +public class DelayMessageLogVisitor implements LogVisitor { + private static final Logger LOGGER = LoggerFactory.getLogger(DelayMessageLogVisitor.class); + + private static final EmptyLogRecord EMPTY_LOG_RECORD = new EmptyLogRecord(); + private static final int MIN_RECORD_BYTES = 13; + + private final AtomicInteger visitedBufferSize = new AtomicInteger(0); + + private LogSegment currentSegment; + private SegmentBuffer currentBuffer; + private long startOffset; + + public DelayMessageLogVisitor(LogManager logManager, final Long startOffset) { + long initialOffset = initialOffset(logManager, startOffset); + this.currentSegment = logManager.locateSegment(initialOffset); + this.currentBuffer = selectBuffer(initialOffset); + this.startOffset = initialOffset; + } + + public long startOffset() { + return startOffset; + } + + @Override + public Optional nextRecord() { + if (currentBuffer == null) { + return Optional.of(EMPTY_LOG_RECORD); + } + + return readOneRecord(currentBuffer.getBuffer()); + } + + @Override + public long visitedBufferSize() { + if (currentBuffer == null) { + return 0; + } + return visitedBufferSize.get(); + } + + private long initialOffset(final LogManager logManager, final Long originStart) { + long minOffset = logManager.getMinOffset(); + if (originStart < minOffset) { + LOGGER.error("initial delay message log visitor offset less than min offset. start: {}, min: {}", + originStart, minOffset); + return minOffset; + } + + return originStart; + } + + private SegmentBuffer selectBuffer(final long startOffset) { + if (currentSegment == null) { + return null; + } + final int pos = (int) (startOffset % currentSegment.getFileSize()); + return currentSegment.selectSegmentBuffer(pos); + } + + private Optional readOneRecord(final ByteBuffer buffer) { + if (buffer.remaining() < MIN_RECORD_BYTES) { + return Optional.of(EMPTY_LOG_RECORD); + } + + final int startPos = buffer.position(); + long startWroteOffset = startOffset + startPos; + // magic + final int magic = buffer.getInt(); + if (!MagicCodeSupport.isValidMessageLogMagicCode(magic)) { +// visitedBufferSize.set(currentBuffer.getSize()); + return Optional.of(EMPTY_LOG_RECORD); + } + + // attr + final byte attributes = buffer.get(); + //timestamp + buffer.getLong(); + if (attributes == ATTR_SKIP_RECORD.getCode()) { + if (buffer.remaining() < Integer.BYTES) { + return Optional.of(EMPTY_LOG_RECORD); + } + // blank size + final int blankSize = buffer.getInt(); + visitedBufferSize.addAndGet(blankSize + (buffer.position() - startPos)); + return Optional.empty(); + } else if (attributes == ATTR_EMPTY_RECORD.getCode()) { + visitedBufferSize.set(currentBuffer.getSize()); + return Optional.of(EMPTY_LOG_RECORD); + } else { + if (buffer.remaining() < Long.BYTES) { + return Optional.of(EMPTY_LOG_RECORD); + } + // schedule time + long scheduleTime = buffer.getLong(); + if (buffer.remaining() < Long.BYTES) { + return Optional.of(EMPTY_LOG_RECORD); + } + // logical offset + long sequence = buffer.getLong(); + if (buffer.remaining() < Integer.BYTES) { + return Optional.of(EMPTY_LOG_RECORD); + } + // message id size + final int messageIdSize = buffer.getInt(); + if (buffer.remaining() < messageIdSize) { + return Optional.of(EMPTY_LOG_RECORD); + } + final byte[] messageIdBytes = new byte[messageIdSize]; + // message id + buffer.get(messageIdBytes); + + // subject size + final int subjectSize = buffer.getInt(); + if (buffer.remaining() < subjectSize) { + return Optional.of(EMPTY_LOG_RECORD); + } + final byte[] subjectBytes = new byte[subjectSize]; + // subject + buffer.get(subjectBytes); + + if (magic >= MagicCode.MESSAGE_LOG_MAGIC_V2) { + if (buffer.remaining() < Long.BYTES) { + return Optional.of(EMPTY_LOG_RECORD); + } + // crc + buffer.getLong(); + } + + if (buffer.remaining() < Integer.BYTES) { + return Optional.of(EMPTY_LOG_RECORD); + } + // payload size + final int payloadSize = buffer.getInt(); + if (buffer.remaining() < payloadSize) { + return Optional.of(EMPTY_LOG_RECORD); + } + // message body && The new buffer's position will be zero + final ByteBuffer message = buffer.slice(); + message.limit(payloadSize); + buffer.position(buffer.position() + payloadSize); + int recordBytes = buffer.position() - startPos; + visitedBufferSize.addAndGet(recordBytes); + LogRecordHeader header = new LogRecordHeader(new String(subjectBytes, StandardCharsets.UTF_8), new String(messageIdBytes, StandardCharsets.UTF_8), scheduleTime, sequence); + return Optional.of(new MessageLogRecord(header, recordBytes, startWroteOffset, payloadSize, message)); + } + } + + @Override + public void close() { + + } + + public static class EmptyLogRecord implements LogRecord { + + @Override + public String getSubject() { + return null; + } + + @Override + public String getMessageId() { + return null; + } + + @Override + public long getScheduleTime() { + return 0; + } + + @Override + public int getPayloadSize() { + return 0; + } + + @Override + public ByteBuffer getRecord() { + return null; + } + + @Override + public long getStartWroteOffset() { + return 0; + } + + @Override + public int getRecordSize() { + return 0; + } + + @Override + public long getSequence() { + return 0; + } + } +} diff --git a/qmq-delay-server/src/main/java/qunar/tc/qmq/delay/store/visitor/DispatchLogVisitor.java b/qmq-delay-server/src/main/java/qunar/tc/qmq/delay/store/visitor/DispatchLogVisitor.java new file mode 100644 index 00000000..47d8378b --- /dev/null +++ b/qmq-delay-server/src/main/java/qunar/tc/qmq/delay/store/visitor/DispatchLogVisitor.java @@ -0,0 +1,45 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.delay.store.visitor; + +import java.nio.ByteBuffer; +import java.nio.channels.FileChannel; +import java.util.Optional; + +/** + * @author xufeng.deng dennisdxf@gmail.com + * @since 2018-07-15 15:01 + */ +public class DispatchLogVisitor extends AbstractLogVisitor { + private static final int WORKING_SIZE = 1024 * 1024 * 10; + + public DispatchLogVisitor(long from, FileChannel fileChannel) { + super(from, fileChannel, WORKING_SIZE); + } + + @Override + protected Optional readOneRecord(ByteBuffer buffer) { + if (buffer.remaining() < 8) { + return Optional.empty(); + } + int startPos = buffer.position(); + long index = buffer.getLong(); + + buffer.position(startPos + 8); + return Optional.of(index); + } +} diff --git a/qmq-delay-server/src/main/java/qunar/tc/qmq/delay/store/visitor/LogVisitor.java b/qmq-delay-server/src/main/java/qunar/tc/qmq/delay/store/visitor/LogVisitor.java new file mode 100644 index 00000000..d87f1d06 --- /dev/null +++ b/qmq-delay-server/src/main/java/qunar/tc/qmq/delay/store/visitor/LogVisitor.java @@ -0,0 +1,33 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.delay.store.visitor; + +import java.util.Optional; + +/** + * @author xufeng.deng dennisdxf@gmail.com + * @since 2018-07-12 11:36 + */ +public interface LogVisitor { + + Optional nextRecord(); + + long visitedBufferSize(); + + void close(); + +} diff --git a/qmq-delay-server/src/main/java/qunar/tc/qmq/delay/store/visitor/ScheduleIndexVisitor.java b/qmq-delay-server/src/main/java/qunar/tc/qmq/delay/store/visitor/ScheduleIndexVisitor.java new file mode 100644 index 00000000..f2552665 --- /dev/null +++ b/qmq-delay-server/src/main/java/qunar/tc/qmq/delay/store/visitor/ScheduleIndexVisitor.java @@ -0,0 +1,95 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.delay.store.visitor; + +import io.netty.buffer.ByteBuf; +import qunar.tc.qmq.delay.ScheduleIndex; + +import java.nio.ByteBuffer; +import java.nio.channels.FileChannel; +import java.util.Optional; + +/** + * @author xufeng.deng dennisdxf@gmail.com + * @since 2018-07-15 15:01 + */ +public class ScheduleIndexVisitor extends AbstractLogVisitor { + + public ScheduleIndexVisitor(long from, FileChannel fileChannel, int singleMessageLimitSize) { + super(from, fileChannel, singleMessageLimitSize); + } + + @Override + protected Optional readOneRecord(ByteBuffer buffer) { + long curPos = buffer.position(); + + if (buffer.remaining() < Long.BYTES) { + return Optional.empty(); + } + long scheduleTime = buffer.getLong(); + + if (buffer.remaining() < Long.BYTES) { + return Optional.empty(); + } + long sequence = buffer.getLong(); + + if (buffer.remaining() < Integer.BYTES) { + return Optional.empty(); + } + int payloadSize = buffer.getInt(); + + if (buffer.remaining() < Integer.BYTES) { + return Optional.empty(); + } + int messageId = buffer.getInt(); + + if (buffer.remaining() < messageId) { + return Optional.empty(); + } + buffer.position(buffer.position() + messageId); + + if (buffer.remaining() < Integer.BYTES) { + return Optional.empty(); + } + int subject = buffer.getInt(); + + if (buffer.remaining() < subject) { + return Optional.empty(); + } + buffer.position(buffer.position() + subject); + + if (buffer.remaining() < payloadSize) { + return Optional.empty(); + } + + int metaSize = getMetaSize(messageId, subject); + int recordSize = metaSize + payloadSize; + long startOffset = visitedBufferSize(); + buffer.position(Math.toIntExact(curPos + recordSize)); + + return Optional.of(ScheduleIndex.buildIndex(scheduleTime, startOffset, recordSize, sequence)); + } + + private int getMetaSize(int messageId, int subject) { + return 8 + 8 + + 4 + + 4 + + 4 + + messageId + + subject; + } +} diff --git a/qmq-delay-server/src/main/java/qunar/tc/qmq/delay/sync/master/AbstractLogSyncWorker.java b/qmq-delay-server/src/main/java/qunar/tc/qmq/delay/sync/master/AbstractLogSyncWorker.java new file mode 100644 index 00000000..e5a839da --- /dev/null +++ b/qmq-delay-server/src/main/java/qunar/tc/qmq/delay/sync/master/AbstractLogSyncWorker.java @@ -0,0 +1,89 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.delay.sync.master; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import qunar.tc.qmq.configuration.DynamicConfig; +import qunar.tc.qmq.delay.DelayLogFacade; +import qunar.tc.qmq.protocol.Datagram; +import qunar.tc.qmq.protocol.RemotingHeader; +import qunar.tc.qmq.store.SegmentBuffer; +import qunar.tc.qmq.sync.DelaySyncRequest; + +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.atomic.AtomicBoolean; + +/** + * @author yunfeng.yang + * @since 2017/8/21 + */ +abstract class AbstractLogSyncWorker implements DelaySyncRequestProcessor.SyncProcessor { + protected static final Logger LOGGER = LoggerFactory.getLogger(AbstractLogSyncWorker.class); + protected final DynamicConfig config; + final DelayLogFacade delayLogFacade; + private final AtomicBoolean TIMER_DEFINED = new AtomicBoolean(false); + ScheduledExecutorService processorTimer; + + AbstractLogSyncWorker(DelayLogFacade delayLogFacade, DynamicConfig config) { + this.delayLogFacade = delayLogFacade; + this.config = config; + } + + @Override + public void process(DelaySyncRequestProcessor.SyncRequestEntry entry) { + final DelaySyncRequest delaySyncRequest = entry.getDelaySyncRequest(); + final SegmentBuffer result = getSyncLog(delaySyncRequest); + if (result == null || result.getSize() <= 0) { + LOGGER.debug("log sync process empty result {}, {}", result, delaySyncRequest); + newTimeout(new DelaySyncRequestProcessor.SyncRequestTimeoutTask(entry, this)); + return; + } + + processSyncLog(entry, result); + } + + void defineTimer(ScheduledExecutorService processorTimer) { + if (TIMER_DEFINED.compareAndSet(false, true)) { + this.processorTimer = processorTimer; + return; + } + + LOGGER.error("message log sync worker timer defined more than once"); + } + + @Override + public void processTimeout(DelaySyncRequestProcessor.SyncRequestEntry entry) { + final DelaySyncRequest syncRequest = entry.getDelaySyncRequest(); + final SegmentBuffer result = getSyncLog(syncRequest); + + if (result == null || result.getSize() <= 0) { + entry.getCtx().writeAndFlush(resolveResult(syncRequest, entry.getRequestHeader())); + return; + } + + processSyncLog(entry, result); + } + + protected abstract void newTimeout(final DelaySyncRequestProcessor.SyncRequestTimeoutTask task); + + protected abstract Datagram resolveResult(final DelaySyncRequest syncRequest, final RemotingHeader header); + + protected abstract void processSyncLog(final DelaySyncRequestProcessor.SyncRequestEntry entry, final SegmentBuffer result); + + protected abstract SegmentBuffer getSyncLog(final DelaySyncRequest delaySyncRequest); +} diff --git a/qmq-delay-server/src/main/java/qunar/tc/qmq/delay/sync/master/DelaySyncRequestProcessor.java b/qmq-delay-server/src/main/java/qunar/tc/qmq/delay/sync/master/DelaySyncRequestProcessor.java new file mode 100644 index 00000000..0f89f6dc --- /dev/null +++ b/qmq-delay-server/src/main/java/qunar/tc/qmq/delay/sync/master/DelaySyncRequestProcessor.java @@ -0,0 +1,188 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.delay.sync.master; + +import com.google.common.util.concurrent.ThreadFactoryBuilder; +import io.netty.buffer.ByteBuf; +import io.netty.channel.ChannelHandlerContext; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import qunar.tc.qmq.common.Disposable; +import qunar.tc.qmq.configuration.DynamicConfig; +import qunar.tc.qmq.delay.DelayLogFacade; +import qunar.tc.qmq.netty.NettyRequestProcessor; +import qunar.tc.qmq.protocol.CommandCode; +import qunar.tc.qmq.protocol.Datagram; +import qunar.tc.qmq.protocol.RemotingCommand; +import qunar.tc.qmq.protocol.RemotingHeader; +import qunar.tc.qmq.sync.DelaySyncRequest; +import qunar.tc.qmq.sync.SyncType; +import qunar.tc.qmq.util.RemotingBuilder; + +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.*; + +/** + * @author xufeng.deng dennisdxf@gmail.com + * @since 2018-08-10 11:33 + */ +public class DelaySyncRequestProcessor implements NettyRequestProcessor, Disposable { + private static final Logger LOGGER = LoggerFactory.getLogger(DelaySyncRequestProcessor.class); + + private final ExecutorService syncProcessPool; + private final Map processorMap; + private final MessageLogSyncWorker messageLogSyncWorker; + private final ScheduledExecutorService processorTimer; + + DelaySyncRequestProcessor(DelayLogFacade delayLogFacade, DynamicConfig config) { + this.processorMap = new HashMap<>(); + this.messageLogSyncWorker = new MessageLogSyncWorker(delayLogFacade, config); + final DispatchLogSyncWorker dispatchLogSyncWorker = new DispatchLogSyncWorker(delayLogFacade, config); + final HeartbeatSyncWorker heartbeatSyncWorker = new HeartbeatSyncWorker(delayLogFacade); + + this.processorMap.put(SyncType.message.getCode(), this.messageLogSyncWorker); + this.processorMap.put(SyncType.dispatch.getCode(), dispatchLogSyncWorker); + this.processorMap.put(SyncType.heartbeat.getCode(), heartbeatSyncWorker); + + int workerSize = processorMap.size(); + this.syncProcessPool = Executors.newFixedThreadPool(workerSize, + new ThreadFactoryBuilder().setNameFormat("delay-master-sync-%d").build()); + this.processorTimer = Executors.newScheduledThreadPool(workerSize - 1, + new ThreadFactoryBuilder().setNameFormat("delay-processor-timer-%d").build()); + + this.messageLogSyncWorker.defineTimer(processorTimer); + dispatchLogSyncWorker.defineTimer(processorTimer); + } + + @Override + public void destroy() { + syncProcessPool.shutdown(); + if (null != processorTimer) { + processorTimer.shutdown(); + try { + processorTimer.awaitTermination(10, TimeUnit.SECONDS); + } catch (InterruptedException e) { + LOGGER.error("Shutdown processorTimer interrupted."); + } + } + } + + @Override + public CompletableFuture processRequest(ChannelHandlerContext ctx, RemotingCommand request) { + DelaySyncRequest delaySyncRequest = deserializeSyncRequest(request); + int logType = delaySyncRequest.getSyncType(); + SyncProcessor processor = processorMap.get(logType); + if (null == processor) { + LOGGER.error("unknown log type {}", logType); + final Datagram datagram = RemotingBuilder.buildResponseDatagram(CommandCode.BROKER_ERROR, request.getHeader(), null); + return CompletableFuture.completedFuture(datagram); + } + + SyncRequestEntry entry = new SyncRequestEntry(ctx, request.getHeader(), delaySyncRequest); + syncProcessPool.submit(new SyncRequestProcessTask(entry, processor)); + return null; + } + + private DelaySyncRequest deserializeSyncRequest(RemotingCommand request) { + ByteBuf body = request.getBody(); + int logType = body.readByte(); + long messageLogOffset = body.readLong(); + int dispatchLogSegmentBaseOffset = body.readInt(); + long dispatchLogOffset = body.readLong(); + int lastDispatchLogBaseOffset = body.readInt(); + long lastDispatchLogOffset = body.readLong(); + return new DelaySyncRequest(messageLogOffset, dispatchLogSegmentBaseOffset, dispatchLogOffset, lastDispatchLogBaseOffset, lastDispatchLogOffset, logType); + } + + void registerSyncEvent(Object listener) { + this.messageLogSyncWorker.registerSyncEvent(listener); + } + + @Override + public boolean rejectRequest() { + return false; + } + + interface SyncProcessor { + void process(SyncRequestEntry entry); + + void processTimeout(SyncRequestEntry entry); + } + + static class SyncRequestEntry { + private final ChannelHandlerContext ctx; + private final RemotingHeader requestHeader; + private final DelaySyncRequest delaySyncRequest; + + SyncRequestEntry(ChannelHandlerContext ctx, RemotingHeader requestHeader, DelaySyncRequest delaySyncRequest) { + this.ctx = ctx; + this.requestHeader = requestHeader; + this.delaySyncRequest = delaySyncRequest; + } + + ChannelHandlerContext getCtx() { + return ctx; + } + + RemotingHeader getRequestHeader() { + return requestHeader; + } + + DelaySyncRequest getDelaySyncRequest() { + return delaySyncRequest; + } + } + + private static class SyncRequestProcessTask implements Runnable { + private final SyncRequestEntry entry; + private final SyncProcessor processor; + + private SyncRequestProcessTask(SyncRequestEntry syncRequestEntry, SyncProcessor processor) { + this.entry = syncRequestEntry; + this.processor = processor; + } + + @Override + public void run() { + try { + processor.process(entry); + } catch (Exception e) { + LOGGER.error("process sync request error", e); + } + } + } + + static class SyncRequestTimeoutTask implements Runnable { + private final SyncRequestEntry entry; + private final SyncProcessor processor; + + SyncRequestTimeoutTask(SyncRequestEntry entry, SyncProcessor processor) { + this.entry = entry; + this.processor = processor; + } + + @Override + public void run() { + try { + processor.processTimeout(entry); + } catch (Exception e) { + LOGGER.error("process sync timeout request error", e); + } + } + } +} diff --git a/qmq-delay-server/src/main/java/qunar/tc/qmq/delay/sync/master/DispatchLogSyncWorker.java b/qmq-delay-server/src/main/java/qunar/tc/qmq/delay/sync/master/DispatchLogSyncWorker.java new file mode 100644 index 00000000..3994959d --- /dev/null +++ b/qmq-delay-server/src/main/java/qunar/tc/qmq/delay/sync/master/DispatchLogSyncWorker.java @@ -0,0 +1,187 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.delay.sync.master; + +import io.netty.buffer.ByteBuf; +import qunar.tc.qmq.configuration.DynamicConfig; +import qunar.tc.qmq.delay.DelayLogFacade; +import qunar.tc.qmq.delay.base.SegmentBufferExtend; +import qunar.tc.qmq.protocol.CommandCode; +import qunar.tc.qmq.protocol.Datagram; +import qunar.tc.qmq.protocol.PayloadHolder; +import qunar.tc.qmq.protocol.RemotingHeader; +import qunar.tc.qmq.store.SegmentBuffer; +import qunar.tc.qmq.sync.DelaySyncRequest; +import qunar.tc.qmq.util.RemotingBuilder; + +import java.nio.ByteBuffer; +import java.util.concurrent.TimeUnit; + +import static qunar.tc.qmq.delay.store.log.ScheduleOffsetResolver.resolveSegment; + +/** + * @author xufeng.deng dennisdxf@gmail.com + * @since 2018-08-10 16:13 + */ +public class DispatchLogSyncWorker extends AbstractLogSyncWorker { + + private static final int GREENWICH_MEAN_FIRST_YEAR = 1970010108; + + public DispatchLogSyncWorker(DelayLogFacade delayLogFacade, DynamicConfig config) { + super(delayLogFacade, config); + } + + @Override + protected void newTimeout(DelaySyncRequestProcessor.SyncRequestTimeoutTask task) { + final long timeout = config.getLong("dispatch.sync.timeout.ms", 10L); + processorTimer.schedule(task, timeout, TimeUnit.MILLISECONDS); + } + + @Override + protected SegmentBuffer getSyncLog(DelaySyncRequest delaySyncRequest) { + int nowSegment = resolveSegment(System.currentTimeMillis()); + + SyncOffset syncOffset = resolveOffset(delaySyncRequest); + int segmentBaseOffset = syncOffset.getBaseOffset(); + if (syncOffset.getBaseOffset() == -1) return null; + + long maxDispatchLogOffset = delayLogFacade.getDispatchLogMaxOffset(segmentBaseOffset); + long dispatchLogOffset = syncOffset.getOffset(); + if (dispatchLogOffset < 0) { + LOGGER.warn("dispatch sync illegal param, baseOffset {} dispatchLogOffset {}", segmentBaseOffset, dispatchLogOffset); + dispatchLogOffset = 0; + } + + if (segmentBaseOffset < nowSegment && dispatchLogOffset >= maxDispatchLogOffset) { + if (maxDispatchLogOffset == 0) return null; + + int nextSegmentBaseOffset = delayLogFacade.higherDispatchLogBaseOffset(segmentBaseOffset); + if (nextSegmentBaseOffset < 0) { + return null; + } + + // sync next + segmentBaseOffset = nextSegmentBaseOffset; + dispatchLogOffset = 0; + } + + SegmentBuffer result = delayLogFacade.getDispatchLogs(segmentBaseOffset, dispatchLogOffset); + + // previous segment(< now) size may be 0, then skip, wait until timeout + while (result != null && result.getSize() <= 0) { + segmentBaseOffset = delayLogFacade.higherDispatchLogBaseOffset(segmentBaseOffset); + if (segmentBaseOffset < 0 || segmentBaseOffset > nowSegment) { + return null; + } + result = delayLogFacade.getDispatchLogs(segmentBaseOffset, dispatchLogOffset); + } + + return result; + } + + private SyncOffset resolveOffset(final DelaySyncRequest delaySyncRequest) { + int segmentBaseOffset = delaySyncRequest.getDispatchSegmentBaseOffset(); + long offset = delaySyncRequest.getDispatchLogOffset(); + int lastSegmentBaseOffset = delaySyncRequest.getLastDispatchSegmentBaseOffset(); + long lastOffset = delaySyncRequest.getLastDispatchSegmentOffset(); + + // sync the first time + if (segmentBaseOffset < GREENWICH_MEAN_FIRST_YEAR) { + return new SyncOffset(delayLogFacade.higherDispatchLogBaseOffset(segmentBaseOffset), 0); + } + + // only one dispatch segment + if (lastSegmentBaseOffset < GREENWICH_MEAN_FIRST_YEAR) { + return new SyncOffset(segmentBaseOffset, offset); + } + + long lastSegmentMaxOffset = delayLogFacade.getDispatchLogMaxOffset(lastSegmentBaseOffset); + // sync last + if (lastOffset < lastSegmentMaxOffset) { + return new SyncOffset(lastSegmentBaseOffset, lastOffset); + } + + // sync current, probably + return new SyncOffset(segmentBaseOffset, offset); + } + + @Override + protected void processSyncLog(DelaySyncRequestProcessor.SyncRequestEntry entry, SegmentBuffer result) { + int batchSize = config.getInt("sync.batch.size", 100000); + long startOffset = result.getStartOffset(); + ByteBuffer buffer = result.getBuffer(); + int size = result.getSize(); + int segmentBaseOffset = ((SegmentBufferExtend) result).getBaseOffset(); + + if (size > batchSize) { + buffer.limit(batchSize); + size = batchSize; + } + final Datagram datagram = RemotingBuilder.buildResponseDatagram(CommandCode.SUCCESS, entry.getRequestHeader() + , new SyncDispatchLogPayloadHolder(size, startOffset, segmentBaseOffset, buffer)); + entry.getCtx().writeAndFlush(datagram); + } + + @Override + protected Datagram resolveResult(DelaySyncRequest syncRequest, RemotingHeader header) { + long offset = syncRequest.getDispatchLogOffset(); + SyncDispatchLogPayloadHolder payloadHolder = new SyncDispatchLogPayloadHolder(0, offset, syncRequest.getDispatchSegmentBaseOffset(), null); + return RemotingBuilder.buildResponseDatagram(CommandCode.SUCCESS, header, payloadHolder); + } + + public static class SyncDispatchLogPayloadHolder implements PayloadHolder { + private final int size; + private final long startOffset; + private final int segmentBaseOffset; + private final ByteBuffer buffer; + + public SyncDispatchLogPayloadHolder(int size, long startOffset, int segmentBaseOffset, ByteBuffer buffer) { + this.size = size; + this.startOffset = startOffset; + this.segmentBaseOffset = segmentBaseOffset; + this.buffer = buffer; + } + + @Override + public void writeBody(ByteBuf out) { + out.writeInt(size); + out.writeLong(startOffset); + out.writeInt(segmentBaseOffset); + if (buffer != null) { + out.writeBytes(buffer); + } + } + } + + private static class SyncOffset { + private final int baseOffset; + private final long offset; + + SyncOffset(int baseOffset, long offset) { + this.baseOffset = baseOffset; + this.offset = offset; + } + + public int getBaseOffset() { + return baseOffset; + } + + public long getOffset() { + return offset; + } + } +} diff --git a/qmq-delay-server/src/main/java/qunar/tc/qmq/delay/sync/master/HeartbeatSyncWorker.java b/qmq-delay-server/src/main/java/qunar/tc/qmq/delay/sync/master/HeartbeatSyncWorker.java new file mode 100644 index 00000000..f07f55c2 --- /dev/null +++ b/qmq-delay-server/src/main/java/qunar/tc/qmq/delay/sync/master/HeartbeatSyncWorker.java @@ -0,0 +1,81 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.delay.sync.master; + +import io.netty.buffer.ByteBuf; +import io.netty.channel.ChannelHandlerContext; +import qunar.tc.qmq.delay.DelayLogFacade; +import qunar.tc.qmq.delay.monitor.QMon; +import qunar.tc.qmq.metrics.Metrics; +import qunar.tc.qmq.protocol.CommandCode; +import qunar.tc.qmq.protocol.Datagram; +import qunar.tc.qmq.protocol.PayloadHolder; +import qunar.tc.qmq.sync.SyncType; +import qunar.tc.qmq.util.RemotingBuilder; + +/** + * @author xufeng.deng dennisdxf@gmail.com + * @since 2018-08-10 17:51 + */ +public class HeartbeatSyncWorker implements DelaySyncRequestProcessor.SyncProcessor { + private final DelayLogFacade delayLogFacade; + + private volatile long slaveMessageLogLag = 0; + private volatile long slaveCurrentSegmentDispatchLogLag = 0; + + HeartbeatSyncWorker(DelayLogFacade delayLogFacade) { + this.delayLogFacade = delayLogFacade; + Metrics.gauge("slaveMessageLogLag", () -> (double) slaveMessageLogLag); + Metrics.gauge("slaveCurrentSegmentDispatchLogLag", () -> (double) slaveCurrentSegmentDispatchLogLag); + } + + @Override + public void process(DelaySyncRequestProcessor.SyncRequestEntry entry) { + final ChannelHandlerContext ctx = entry.getCtx(); + final long messageLogMaxOffset = delayLogFacade.getMessageLogMaxOffset(); + final long dispatchLogMaxOffset = delayLogFacade.getDispatchLogMaxOffset(entry.getDelaySyncRequest().getDispatchSegmentBaseOffset()); + + slaveMessageLogLag = messageLogMaxOffset - entry.getDelaySyncRequest().getMessageLogOffset(); + slaveCurrentSegmentDispatchLogLag = dispatchLogMaxOffset - entry.getDelaySyncRequest().getDispatchLogOffset(); + QMon.slaveSyncLogOffset(SyncType.message.name(), slaveMessageLogLag); + QMon.slaveSyncLogOffset(SyncType.dispatch.name(), slaveCurrentSegmentDispatchLogLag); + final HeartBeatPayloadHolder heartBeatPayloadHolder = new HeartBeatPayloadHolder(messageLogMaxOffset, dispatchLogMaxOffset); + final Datagram datagram = RemotingBuilder.buildResponseDatagram(CommandCode.SUCCESS, entry.getRequestHeader(), heartBeatPayloadHolder); + ctx.writeAndFlush(datagram); + } + + @Override + public void processTimeout(DelaySyncRequestProcessor.SyncRequestEntry entry) { + + } + + private static class HeartBeatPayloadHolder implements PayloadHolder { + private final long messageLogMaxOffset; + private final long dispatchLogOffset; + + HeartBeatPayloadHolder(long messageLogMaxOffset, long dispatchLogOffset) { + this.messageLogMaxOffset = messageLogMaxOffset; + this.dispatchLogOffset = dispatchLogOffset; + } + + @Override + public void writeBody(ByteBuf out) { + out.writeLong(messageLogMaxOffset); + out.writeLong(dispatchLogOffset); + } + } +} diff --git a/qmq-delay-server/src/main/java/qunar/tc/qmq/delay/sync/master/MasterSyncNettyServer.java b/qmq-delay-server/src/main/java/qunar/tc/qmq/delay/sync/master/MasterSyncNettyServer.java new file mode 100644 index 00000000..0004190d --- /dev/null +++ b/qmq-delay-server/src/main/java/qunar/tc/qmq/delay/sync/master/MasterSyncNettyServer.java @@ -0,0 +1,53 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.delay.sync.master; + +import qunar.tc.qmq.common.Disposable; +import qunar.tc.qmq.configuration.DynamicConfig; +import qunar.tc.qmq.delay.DelayLogFacade; +import qunar.tc.qmq.netty.DefaultConnectionEventHandler; +import qunar.tc.qmq.netty.NettyServer; +import qunar.tc.qmq.protocol.CommandCode; + +/** + * @author yunfeng.yang + * @since 2017/8/19 + */ +public class MasterSyncNettyServer implements Disposable { + private final NettyServer nettyServer; + private final DelaySyncRequestProcessor syncRequestProcessor; + + public MasterSyncNettyServer(final DynamicConfig config, final DelayLogFacade delayLogFacade) { + final Integer port = config.getInt("sync.port", 20802); + this.nettyServer = new NettyServer("sync", 4, port, new DefaultConnectionEventHandler("delay-sync")); + this.syncRequestProcessor = new DelaySyncRequestProcessor(delayLogFacade, config); + } + + public void registerSyncEvent(Object listener) { + syncRequestProcessor.registerSyncEvent(listener); + } + + public void start() { + nettyServer.registerProcessor(CommandCode.SYNC_LOG_REQUEST, syncRequestProcessor); + nettyServer.start(); + } + + @Override + public void destroy() { + nettyServer.destroy(); + } +} diff --git a/qmq-delay-server/src/main/java/qunar/tc/qmq/delay/sync/master/MessageLogSyncWorker.java b/qmq-delay-server/src/main/java/qunar/tc/qmq/delay/sync/master/MessageLogSyncWorker.java new file mode 100644 index 00000000..3da8f8ae --- /dev/null +++ b/qmq-delay-server/src/main/java/qunar/tc/qmq/delay/sync/master/MessageLogSyncWorker.java @@ -0,0 +1,123 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.delay.sync.master; + +import com.google.common.eventbus.AsyncEventBus; +import com.google.common.util.concurrent.ThreadFactoryBuilder; +import io.netty.buffer.ByteBuf; +import qunar.tc.qmq.common.Disposable; +import qunar.tc.qmq.configuration.DynamicConfig; +import qunar.tc.qmq.delay.DelayLogFacade; +import qunar.tc.qmq.protocol.CommandCode; +import qunar.tc.qmq.protocol.Datagram; +import qunar.tc.qmq.protocol.PayloadHolder; +import qunar.tc.qmq.protocol.RemotingHeader; +import qunar.tc.qmq.store.SegmentBuffer; +import qunar.tc.qmq.sync.DelaySyncRequest; +import qunar.tc.qmq.util.RemotingBuilder; + +import java.nio.ByteBuffer; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; + +/** + * @author yunfeng.yang + * @since 2017/8/19 + */ +public class MessageLogSyncWorker extends AbstractLogSyncWorker implements Disposable { + private final AsyncEventBus messageLogSyncEventBus; + private final ExecutorService dispatchExecutor; + + public MessageLogSyncWorker(DelayLogFacade delayLogFacade, DynamicConfig config) { + super(delayLogFacade, config); + this.dispatchExecutor = Executors.newSingleThreadExecutor(new ThreadFactoryBuilder().setNameFormat("heart-event-bus-%d").build()); + this.messageLogSyncEventBus = new AsyncEventBus(dispatchExecutor); + } + + @Override + public void newTimeout(DelaySyncRequestProcessor.SyncRequestTimeoutTask task) { + final long timeout = config.getLong("message.sync.timeout.ms", 10L); + processorTimer.schedule(task, timeout, TimeUnit.MILLISECONDS); + } + + @Override + public SegmentBuffer getSyncLog(DelaySyncRequest syncRequest) { + messageLogSyncEventBus.post(syncRequest); + long startSyncOffset = syncRequest.getMessageLogOffset(); + if (startSyncOffset <= 0) { + startSyncOffset = delayLogFacade.getMessageLogMinOffset(); + } + + return delayLogFacade.getMessageLogs(startSyncOffset); + } + + @Override + protected void processSyncLog(DelaySyncRequestProcessor.SyncRequestEntry entry, SegmentBuffer result) { + final int batchSize = config.getInt("sync.batch.size", 100000); + final long startOffset = result.getStartOffset(); + final ByteBuffer buffer = result.getBuffer(); + int size = result.getSize(); + + if (size > batchSize) { + buffer.limit(batchSize); + size = batchSize; + } + final Datagram datagram = RemotingBuilder.buildResponseDatagram(CommandCode.SUCCESS, entry.getRequestHeader() + , new SyncLogPayloadHolder(size, startOffset, buffer)); + entry.getCtx().writeAndFlush(datagram); + } + + @Override + protected Datagram resolveResult(DelaySyncRequest syncRequest, RemotingHeader header) { + long offset = syncRequest.getMessageLogOffset(); + SyncLogPayloadHolder payloadHolder = new SyncLogPayloadHolder(0, offset, null); + return RemotingBuilder.buildResponseDatagram(CommandCode.SUCCESS, header, payloadHolder); + } + + void registerSyncEvent(Object listener) { + messageLogSyncEventBus.register(listener); + } + + @Override + public void destroy() { + if (dispatchExecutor != null) { + dispatchExecutor.shutdown(); + } + } + + private static class SyncLogPayloadHolder implements PayloadHolder { + private final int size; + private final long startOffset; + private final ByteBuffer buffer; + + SyncLogPayloadHolder(int size, long startOffset, ByteBuffer buffer) { + this.size = size; + this.startOffset = startOffset; + this.buffer = buffer; + } + + @Override + public void writeBody(ByteBuf out) { + out.writeInt(size); + out.writeLong(startOffset); + if (buffer != null) { + out.writeBytes(buffer); + } + } + } +} diff --git a/qmq-delay-server/src/main/java/qunar/tc/qmq/delay/sync/slave/DispatchLogSyncProcessor.java b/qmq-delay-server/src/main/java/qunar/tc/qmq/delay/sync/slave/DispatchLogSyncProcessor.java new file mode 100644 index 00000000..c9a50e56 --- /dev/null +++ b/qmq-delay-server/src/main/java/qunar/tc/qmq/delay/sync/slave/DispatchLogSyncProcessor.java @@ -0,0 +1,63 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.delay.sync.slave; + +import io.netty.buffer.ByteBuf; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import qunar.tc.qmq.delay.DelayLogFacade; +import qunar.tc.qmq.protocol.Datagram; +import qunar.tc.qmq.sync.DelaySyncRequest; +import qunar.tc.qmq.sync.SyncType; + +/** + * @author xufeng.deng dennisdxf@gmail.com + * @since 2018-08-13 11:04 + */ +public class DispatchLogSyncProcessor implements SyncLogProcessor { + private static final Logger LOGGER = LoggerFactory.getLogger(DispatchLogSyncProcessor.class); + + private final DelayLogFacade facade; + + DispatchLogSyncProcessor(DelayLogFacade facade) { + this.facade = facade; + } + + @Override + public void process(Datagram syncData) { + ByteBuf body = syncData.getBody(); + int size = body.readInt(); + if (size == 0) { + LOGGER.debug("sync dispatch log data empty"); + return; + } + + long startOffset = body.readLong(); + int baseOffset = body.readInt(); + appendLogs(startOffset, baseOffset, body); + } + + private void appendLogs(long startOffset, int baseOffset, ByteBuf body) { + facade.appendDispatchLogData(startOffset, baseOffset, body.nioBuffer()); + } + + @Override + public DelaySyncRequest getRequest() { + DelaySyncRequest.DispatchLogSyncRequest dispatchRequest = facade.getDispatchLogSyncMaxRequest(); + return new DelaySyncRequest(-1, dispatchRequest, SyncType.dispatch.getCode()); + } +} diff --git a/qmq-delay-server/src/main/java/qunar/tc/qmq/delay/sync/slave/HeartBeatProcessor.java b/qmq-delay-server/src/main/java/qunar/tc/qmq/delay/sync/slave/HeartBeatProcessor.java new file mode 100644 index 00000000..0e1e3309 --- /dev/null +++ b/qmq-delay-server/src/main/java/qunar/tc/qmq/delay/sync/slave/HeartBeatProcessor.java @@ -0,0 +1,60 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.delay.sync.slave; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import qunar.tc.qmq.delay.DelayLogFacade; +import qunar.tc.qmq.protocol.Datagram; +import qunar.tc.qmq.sync.DelaySyncRequest; +import qunar.tc.qmq.sync.SyncType; + +import java.util.concurrent.TimeUnit; + +import static qunar.tc.qmq.constants.BrokerConstants.DEFAULT_HEARTBEAT_SLEEP_TIMEOUT_MS; + +/** + * @author xufeng.deng dennisdxf@gmail.com + * @since 2018-08-13 10:23 + */ +public class HeartBeatProcessor implements SyncLogProcessor { + private static final Logger LOGGER = LoggerFactory.getLogger(HeartBeatProcessor.class); + + private final DelayLogFacade facade; + private final long sleepTimeoutMs; + + HeartBeatProcessor(DelayLogFacade facade) { + this.facade = facade; + this.sleepTimeoutMs = DEFAULT_HEARTBEAT_SLEEP_TIMEOUT_MS; + } + + @Override + public void process(Datagram syncData) { + try { + TimeUnit.MILLISECONDS.sleep(sleepTimeoutMs); + } catch (InterruptedException e) { + LOGGER.error("delay slaver heart beat sleep error", e); + } + } + + @Override + public DelaySyncRequest getRequest() { + long messageLogMaxOffset = facade.getMessageLogMaxOffset(); + DelaySyncRequest.DispatchLogSyncRequest dispatchRequest = facade.getDispatchLogSyncMaxRequest(); + return new DelaySyncRequest(messageLogMaxOffset, dispatchRequest, SyncType.heartbeat.getCode()); + } +} diff --git a/qmq-delay-server/src/main/java/qunar/tc/qmq/delay/sync/slave/MessageLogSyncProcessor.java b/qmq-delay-server/src/main/java/qunar/tc/qmq/delay/sync/slave/MessageLogSyncProcessor.java new file mode 100644 index 00000000..37bf17aa --- /dev/null +++ b/qmq-delay-server/src/main/java/qunar/tc/qmq/delay/sync/slave/MessageLogSyncProcessor.java @@ -0,0 +1,61 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.delay.sync.slave; + +import io.netty.buffer.ByteBuf; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import qunar.tc.qmq.delay.DelayLogFacade; +import qunar.tc.qmq.protocol.Datagram; +import qunar.tc.qmq.sync.DelaySyncRequest; +import qunar.tc.qmq.sync.SyncType; + +/** + * @author xufeng.deng dennisdxf@gmail.com + * @since 2018-08-13 10:52 + */ +public class MessageLogSyncProcessor implements SyncLogProcessor { + private static final Logger LOGGER = LoggerFactory.getLogger(MessageLogSyncProcessor.class); + + private final DelayLogFacade facade; + + MessageLogSyncProcessor(DelayLogFacade facade) { + this.facade = facade; + } + + @Override + public void process(Datagram syncData) { + final ByteBuf body = syncData.getBody(); + final int size = body.readInt(); + if (size == 0) { + LOGGER.debug("sync message log data empty"); + return; + } + final long startOffset = body.readLong(); + appendLogs(startOffset, body); + } + + private void appendLogs(long startOffset, ByteBuf body) { + facade.appendMessageLogData(startOffset, body.nioBuffer()); + } + + @Override + public DelaySyncRequest getRequest() { + long messageLogMaxOffset = facade.getMessageLogMaxOffset(); + return new DelaySyncRequest(messageLogMaxOffset, null, SyncType.message.getCode()); + } +} diff --git a/qmq-delay-server/src/main/java/qunar/tc/qmq/delay/sync/slave/SlaveSynchronizer.java b/qmq-delay-server/src/main/java/qunar/tc/qmq/delay/sync/slave/SlaveSynchronizer.java new file mode 100644 index 00000000..ba894731 --- /dev/null +++ b/qmq-delay-server/src/main/java/qunar/tc/qmq/delay/sync/slave/SlaveSynchronizer.java @@ -0,0 +1,162 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.delay.sync.slave; + +import com.google.common.util.concurrent.RateLimiter; +import io.netty.buffer.ByteBuf; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import qunar.tc.qmq.common.Disposable; +import qunar.tc.qmq.configuration.DynamicConfig; +import qunar.tc.qmq.configuration.DynamicConfigLoader; +import qunar.tc.qmq.delay.DelayLogFacade; +import qunar.tc.qmq.metrics.Metrics; +import qunar.tc.qmq.metrics.QmqTimer; +import qunar.tc.qmq.netty.NettyClientConfig; +import qunar.tc.qmq.netty.client.NettyClient; +import qunar.tc.qmq.netty.exception.ClientSendException; +import qunar.tc.qmq.netty.exception.RemoteTimeoutException; +import qunar.tc.qmq.protocol.CommandCode; +import qunar.tc.qmq.protocol.Datagram; +import qunar.tc.qmq.protocol.PayloadHolder; +import qunar.tc.qmq.sync.DelaySyncRequest; +import qunar.tc.qmq.sync.SlaveSyncSender; +import qunar.tc.qmq.sync.SyncType; +import qunar.tc.qmq.util.RemotingBuilder; + +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.TimeUnit; + +/** + * @author xufeng.deng dennisdxf@gmail.com + * @since 2018-08-13 10:12 + */ +public class SlaveSynchronizer implements Disposable { + private static final Logger LOGGER = LoggerFactory.getLogger(SlaveSynchronizer.class); + + private static final RateLimiter LOG_LIMITER = RateLimiter.create(0.2); + + private final String masterAddress; + private final SlaveSyncSender slaveSyncSender; + private final Map slaveSyncTasks; + + public SlaveSynchronizer(final String masterAddress, final DynamicConfig config, final DelayLogFacade facade) { + this.masterAddress = masterAddress; + final NettyClient client = NettyClient.getClient(); + client.start(new NettyClientConfig()); + this.slaveSyncSender = new SlaveSyncSender(config, client); + this.slaveSyncTasks = new HashMap<>(); + + SyncLogProcessor messageLogSyncProcessor = new MessageLogSyncProcessor(facade); + SlaveSyncTask messageLogSyncTask = new SlaveSyncTask(messageLogSyncProcessor); + this.slaveSyncTasks.put(SyncType.message, messageLogSyncTask); + + SyncLogProcessor dispatchLogSyncProcessor = new DispatchLogSyncProcessor(facade); + SlaveSyncTask dispatchLogSyncTask = new SlaveSyncTask(dispatchLogSyncProcessor); + this.slaveSyncTasks.put(SyncType.dispatch, dispatchLogSyncTask); + + SyncLogProcessor heartBeatProcessor = new HeartBeatProcessor(facade); + SlaveSyncTask heartBeatTask = new SlaveSyncTask(heartBeatProcessor); + this.slaveSyncTasks.put(SyncType.heartbeat, heartBeatTask); + } + + public void startSync() { + for (final Map.Entry entry : slaveSyncTasks.entrySet()) { + new Thread(entry.getValue(), "delay-sync-task-" + entry.getKey().name()).start(); + LOGGER.info("slave {} synchronizer started. ", entry.getKey()); + } + } + + @Override + public void destroy() { + slaveSyncTasks.forEach((syncType, slaveSyncTask) -> slaveSyncTask.shutdown()); + } + + private class SlaveSyncTask implements Runnable { + private final SyncLogProcessor processor; + private final String processorName; + + private volatile boolean running = true; + + SlaveSyncTask(SyncLogProcessor processor) { + this.processor = processor; + this.processorName = processor.getClass().getSimpleName(); + } + + void shutdown() { + running = false; + } + + @Override + public void run() { + final long start = System.currentTimeMillis(); + while (running) { + try { + sync(); + } catch (Throwable e) { + if (LOG_LIMITER.tryAcquire()) { + LOGGER.error("sync data from master error", e); + } + } finally { + final QmqTimer timer = Metrics.timer("SyncTask.ExecTimer", new String[]{"processor"}, new String[]{processorName}); + timer.update(System.currentTimeMillis() - start, TimeUnit.MILLISECONDS); + } + } + } + + private void sync() throws InterruptedException, RemoteTimeoutException, ClientSendException { + syncFromMaster(newSyncRequest()); + } + + private Datagram newSyncRequest() { + final DelaySyncRequest request = processor.getRequest(); + final SyncRequestPayloadHolder holder = new SyncRequestPayloadHolder(request); + return RemotingBuilder.buildRequestDatagram(CommandCode.SYNC_LOG_REQUEST, holder); + } + + private void syncFromMaster(Datagram request) throws InterruptedException, RemoteTimeoutException, ClientSendException { + Datagram response = null; + try { + response = slaveSyncSender.send(masterAddress, request); + processor.process(response); + } finally { + if (response != null) { + response.release(); + } + } + } + } + + private class SyncRequestPayloadHolder implements PayloadHolder { + private final DelaySyncRequest request; + + SyncRequestPayloadHolder(DelaySyncRequest request) { + this.request = request; + } + + @Override + public void writeBody(ByteBuf out) { + out.writeByte(request.getSyncType()); + out.writeLong(request.getMessageLogOffset()); + out.writeInt(request.getDispatchSegmentBaseOffset()); + out.writeLong(request.getDispatchLogOffset()); + out.writeInt(request.getLastDispatchSegmentBaseOffset()); + out.writeLong(request.getLastDispatchSegmentOffset()); + } + } +} diff --git a/qmq-delay-server/src/main/java/qunar/tc/qmq/delay/sync/slave/SyncLogProcessor.java b/qmq-delay-server/src/main/java/qunar/tc/qmq/delay/sync/slave/SyncLogProcessor.java new file mode 100644 index 00000000..4e3312eb --- /dev/null +++ b/qmq-delay-server/src/main/java/qunar/tc/qmq/delay/sync/slave/SyncLogProcessor.java @@ -0,0 +1,29 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.delay.sync.slave; + +import qunar.tc.qmq.protocol.Datagram; + +/** + * @author yunfeng.yang + * @since 2017/8/18 + */ +public interface SyncLogProcessor { + void process(Datagram syncData); + + T getRequest(); +} diff --git a/qmq-delay-server/src/main/java/qunar/tc/qmq/delay/wheel/HashedWheelTimer.java b/qmq-delay-server/src/main/java/qunar/tc/qmq/delay/wheel/HashedWheelTimer.java new file mode 100644 index 00000000..6d73cf8b --- /dev/null +++ b/qmq-delay-server/src/main/java/qunar/tc/qmq/delay/wheel/HashedWheelTimer.java @@ -0,0 +1,497 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ +package qunar.tc.qmq.delay.wheel; + +import io.netty.buffer.ByteBuf; +import io.netty.util.internal.PlatformDependent; +import io.netty.util.internal.StringUtil; + +import java.util.Collections; +import java.util.HashSet; +import java.util.Queue; +import java.util.Set; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ThreadFactory; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicIntegerFieldUpdater; + +@SuppressWarnings("all") +public class HashedWheelTimer { + private static final AtomicIntegerFieldUpdater WORKER_STATE_UPDATER; + + static { + AtomicIntegerFieldUpdater workerStateUpdater = + PlatformDependent.newAtomicIntegerFieldUpdater(HashedWheelTimer.class, "workerState"); + if (workerStateUpdater == null) { + workerStateUpdater = AtomicIntegerFieldUpdater.newUpdater(HashedWheelTimer.class, "workerState"); + } + WORKER_STATE_UPDATER = workerStateUpdater; + } + + private final Worker worker = new Worker(); + private final Thread workerThread; + + public static final int WORKER_STATE_INIT = 0; + public static final int WORKER_STATE_STARTED = 1; + public static final int WORKER_STATE_SHUTDOWN = 2; + @SuppressWarnings({"unused", "FieldMayBeFinal", "RedundantFieldInitialization"}) + private volatile int workerState = WORKER_STATE_INIT; // 0 - init, 1 - started, 2 - shut down + + private final long tickDuration; + private final HashedWheelBucket[] wheel; + private final int mask; + private final CountDownLatch startTimeInitialized = new CountDownLatch(1); + private final Queue timeouts = PlatformDependent.newMpscQueue(); + + private volatile long startTime; + private final Processor processor; + + public HashedWheelTimer(ThreadFactory threadFactory, long tickDuration, TimeUnit unit, int ticksPerWheel, Processor processor) { + this.processor = processor; + if (threadFactory == null) { + throw new NullPointerException("threadFactory"); + } + if (unit == null) { + throw new NullPointerException("unit"); + } + if (tickDuration <= 0) { + throw new IllegalArgumentException("tickDuration must be greater than 0: " + tickDuration); + } + if (ticksPerWheel <= 0) { + throw new IllegalArgumentException("ticksPerWheel must be greater than 0: " + ticksPerWheel); + } + + // Normalize ticksPerWheel to power of two and initialize the wheel. + wheel = createWheel(ticksPerWheel); + mask = wheel.length - 1; + + // Convert tickDuration to nanos. + this.tickDuration = unit.toNanos(tickDuration); + + // Prevent overflow. + if (this.tickDuration >= Long.MAX_VALUE / wheel.length) { + throw new IllegalArgumentException(String.format( + "tickDuration: %d (expected: 0 < tickDuration in nanos < %d", + tickDuration, Long.MAX_VALUE / wheel.length)); + } + workerThread = threadFactory.newThread(worker); + } + + private static HashedWheelBucket[] createWheel(int ticksPerWheel) { + if (ticksPerWheel <= 0) { + throw new IllegalArgumentException( + "ticksPerWheel must be greater than 0: " + ticksPerWheel); + } + if (ticksPerWheel > 1073741824) { + throw new IllegalArgumentException( + "ticksPerWheel may not be greater than 2^30: " + ticksPerWheel); + } + + ticksPerWheel = normalizeTicksPerWheel(ticksPerWheel); + HashedWheelBucket[] wheel = new HashedWheelBucket[ticksPerWheel]; + for (int i = 0; i < wheel.length; i++) { + wheel[i] = new HashedWheelBucket(); + } + return wheel; + } + + private static int normalizeTicksPerWheel(int ticksPerWheel) { + int normalizedTicksPerWheel = 1; + while (normalizedTicksPerWheel < ticksPerWheel) { + normalizedTicksPerWheel <<= 1; + } + return normalizedTicksPerWheel; + } + + public void start() { + switch (WORKER_STATE_UPDATER.get(this)) { + case WORKER_STATE_INIT: + if (WORKER_STATE_UPDATER.compareAndSet(this, WORKER_STATE_INIT, WORKER_STATE_STARTED)) { + workerThread.start(); + } + break; + case WORKER_STATE_STARTED: + break; + case WORKER_STATE_SHUTDOWN: + throw new IllegalStateException("cannot be started once stopped"); + default: + throw new Error("Invalid WorkerState"); + } + + // Wait until the startTime is initialized by the worker. + while (startTime == 0) { + try { + startTimeInitialized.await(); + } catch (InterruptedException ignore) { + // Ignore - it will be ready very soon. + } + } + } + + public Set stop() { + if (Thread.currentThread() == workerThread) { + throw new IllegalStateException( + HashedWheelTimer.class.getSimpleName() + + ".stop() cannot be called from "); + } + + if (!WORKER_STATE_UPDATER.compareAndSet(this, WORKER_STATE_STARTED, WORKER_STATE_SHUTDOWN)) { + // workerState can be 0 or 2 at this moment - let it always be 2. + WORKER_STATE_UPDATER.set(this, WORKER_STATE_SHUTDOWN); + + return Collections.emptySet(); + } + + boolean interrupted = false; + while (workerThread.isAlive()) { + workerThread.interrupt(); + try { + workerThread.join(100); + } catch (InterruptedException ignored) { + interrupted = true; + } + } + + if (interrupted) { + Thread.currentThread().interrupt(); + } + + return worker.unprocessedTimeouts(); + } + + public void newTimeout(ByteBuf record, long delay, TimeUnit unit) { + long deadline = System.nanoTime() + unit.toNanos(delay) - startTime; + HashedWheelTimeout timeout = new HashedWheelTimeout(this, record, deadline); + timeouts.add(timeout); + } + + private void expire(ByteBuf record) { + processor.process(record); + } + + private final class Worker implements Runnable { + private final Set unprocessedTimeouts = new HashSet<>(); + + private long tick; + + @Override + public void run() { + // Initialize the startTime. + startTime = System.nanoTime(); + if (startTime == 0) { + // We use 0 as an indicator for the uninitialized value here, so make sure it's not 0 when initialized. + startTime = 1; + } + + // Notify the other threads waiting for the initialization at start(). + startTimeInitialized.countDown(); + + do { + final long deadline = waitForNextTick(); + if (deadline > 0) { + int idx = (int) (tick & mask); + HashedWheelBucket bucket = wheel[idx]; + transferTimeoutsToBuckets(); + bucket.expireTimeouts(deadline); + tick++; + } + } while (WORKER_STATE_UPDATER.get(HashedWheelTimer.this) == WORKER_STATE_STARTED); + + // Fill the unprocessedTimeouts so we can return them from stop() method. + for (HashedWheelBucket bucket : wheel) { + bucket.clearTimeouts(unprocessedTimeouts); + } + for (; ; ) { + HashedWheelTimeout timeout = timeouts.poll(); + if (timeout == null) { + break; + } + + unprocessedTimeouts.add(timeout); + } + } + + private void transferTimeoutsToBuckets() { + // transfer only max. 100000 timeouts per tick to prevent a thread to stale the workerThread when it just + // adds new timeouts in a loop. + for (int i = 0; i < 100000; i++) { + HashedWheelTimeout timeout = timeouts.poll(); + if (timeout == null) { + // all processed + break; + } + + long calculated = timeout.deadline / tickDuration; + timeout.remainingRounds = (calculated - tick) / wheel.length; + + final long ticks = Math.max(calculated, tick); // Ensure we don't schedule for past. + int stopIndex = (int) (ticks & mask); + + HashedWheelBucket bucket = wheel[stopIndex]; + bucket.addTimeout(timeout); + } + } + + /** + * calculate goal nanoTime from startTime and current tick number, + * then wait until that goal has been reached. + * + * @return Long.MIN_VALUE if received a shutdown request, + * current time otherwise (with Long.MIN_VALUE changed by +1) + */ + private long waitForNextTick() { + long deadline = tickDuration * (tick + 1); + + for (; ; ) { + final long currentTime = System.nanoTime() - startTime; + long sleepTimeMs = (deadline - currentTime + 999999) / 1000000; + + if (sleepTimeMs <= 0) { + if (currentTime == Long.MIN_VALUE) { + return -Long.MAX_VALUE; + } else { + return currentTime; + } + } + + // Check if we run on windows, as if thats the case we will need + // to round the sleepTime as workaround for a bug that only affect + // the JVM if it runs on windows. + // + // See https://github.com/netty/netty/issues/356 + if (PlatformDependent.isWindows()) { + sleepTimeMs = sleepTimeMs / 10 * 10; + } + + try { + Thread.sleep(sleepTimeMs); + } catch (InterruptedException ignored) { + if (WORKER_STATE_UPDATER.get(HashedWheelTimer.this) == WORKER_STATE_SHUTDOWN) { + return Long.MIN_VALUE; + } + } + } + } + + public Set unprocessedTimeouts() { + return Collections.unmodifiableSet(unprocessedTimeouts); + } + } + + private static final class HashedWheelTimeout { + + private static final int ST_INIT = 0; + private static final int ST_EXPIRED = 2; + private static final AtomicIntegerFieldUpdater STATE_UPDATER; + + static { + AtomicIntegerFieldUpdater updater = + PlatformDependent.newAtomicIntegerFieldUpdater(HashedWheelTimeout.class, "state"); + if (updater == null) { + updater = AtomicIntegerFieldUpdater.newUpdater(HashedWheelTimeout.class, "state"); + } + STATE_UPDATER = updater; + } + + private final HashedWheelTimer timer; + private final ByteBuf entity; + private final long deadline; + + @SuppressWarnings({"unused", "FieldMayBeFinal", "RedundantFieldInitialization"}) + private volatile int state = ST_INIT; + + // remainingRounds will be calculated and set by Worker.transferTimeoutsToBuckets() before the + // HashedWheelTimeout will be added to the correct HashedWheelBucket. + long remainingRounds; + + // This will be used to chain timeouts in HashedWheelTimerBucket via a double-linked-list. + // As only the workerThread will act on it there is no need for synchronization / volatile. + HashedWheelTimeout next; + HashedWheelTimeout prev; + + // The bucket to which the timeout was added + HashedWheelBucket bucket; + + HashedWheelTimeout(HashedWheelTimer timer, ByteBuf entity, long deadline) { + this.timer = timer; + this.entity = entity; + this.deadline = deadline; + } + + public boolean compareAndSetState(int expected, int state) { + return STATE_UPDATER.compareAndSet(this, expected, state); + } + + public int state() { + return state; + } + + public boolean isExpired() { + return state() == ST_EXPIRED; + } + + public void expire() { + if (!compareAndSetState(ST_INIT, ST_EXPIRED)) { + return; + } + + timer.expire(entity); + } + + @Override + public String toString() { + final long currentTime = System.nanoTime(); + long remaining = deadline - currentTime + timer.startTime; + + StringBuilder buf = new StringBuilder(192) + .append(StringUtil.simpleClassName(this)) + .append('(') + .append("deadline: "); + if (remaining > 0) { + buf.append(remaining) + .append(" ns later"); + } else if (remaining < 0) { + buf.append(-remaining) + .append(" ns ago"); + } else { + buf.append("now"); + } + + return buf.toString(); + } + } + + /** + * Bucket that stores HashedWheelTimeouts. These are stored in a linked-list like datastructure to allow easy + * removal of HashedWheelTimeouts in the middle. Also the HashedWheelTimeout act as nodes themself and so no + * extra object creation is needed. + */ + private static final class HashedWheelBucket { + // Used for the linked-list datastructure + private HashedWheelTimeout head; + private HashedWheelTimeout tail; + + /** + * Add {@link HashedWheelTimeout} to this bucket. + */ + public void addTimeout(HashedWheelTimeout timeout) { + assert timeout.bucket == null; + timeout.bucket = this; + if (head == null) { + head = tail = timeout; + } else { + tail.next = timeout; + timeout.prev = tail; + tail = timeout; + } + } + + /** + * Expire all {@link HashedWheelTimeout}s for the given {@code deadline}. + */ + public void expireTimeouts(long deadline) { + HashedWheelTimeout timeout = head; + + // process all timeouts + while (timeout != null) { + boolean remove = false; + if (timeout.remainingRounds <= 0) { + if (timeout.deadline <= deadline) { + timeout.expire(); + } else { + // The timeout was placed into a wrong slot. This should never happen. + throw new IllegalStateException(String.format( + "timeout.deadline (%d) > deadline (%d)", timeout.deadline, deadline)); + } + remove = true; + } else { + timeout.remainingRounds--; + } + // store reference to next as we may null out timeout.next in the remove block. + HashedWheelTimeout next = timeout.next; + if (remove) { + remove(timeout); + } + timeout = next; + } + } + + public void remove(HashedWheelTimeout timeout) { + HashedWheelTimeout next = timeout.next; + // remove timeout that was either processed or cancelled by updating the linked-list + if (timeout.prev != null) { + timeout.prev.next = next; + } + if (timeout.next != null) { + timeout.next.prev = timeout.prev; + } + + if (timeout == head) { + // if timeout is also the tail we need to adjust the entry too + if (timeout == tail) { + tail = null; + head = null; + } else { + head = next; + } + } else if (timeout == tail) { + // if the timeout is the tail modify the tail to be the prev node. + tail = timeout.prev; + } + // null out prev, next and bucket to allow for GC. + timeout.prev = null; + timeout.next = null; + timeout.bucket = null; + } + + public void clearTimeouts(Set set) { + for (; ; ) { + HashedWheelTimeout timeout = pollTimeout(); + if (timeout == null) { + return; + } + if (timeout.isExpired()) { + continue; + } + set.add(timeout); + } + } + + private HashedWheelTimeout pollTimeout() { + HashedWheelTimeout head = this.head; + if (head == null) { + return null; + } + HashedWheelTimeout next = head.next; + if (next == null) { + tail = this.head = null; + } else { + this.head = next; + next.prev = null; + } + + // null out prev and next to allow for GC. + head.next = null; + head.prev = null; + head.bucket = null; + return head; + } + } + + public interface Processor { + void process(ByteBuf record); + } +} diff --git a/qmq-delay-server/src/main/java/qunar/tc/qmq/delay/wheel/WheelLoadCursor.java b/qmq-delay-server/src/main/java/qunar/tc/qmq/delay/wheel/WheelLoadCursor.java new file mode 100644 index 00000000..a75419ae --- /dev/null +++ b/qmq-delay-server/src/main/java/qunar/tc/qmq/delay/wheel/WheelLoadCursor.java @@ -0,0 +1,123 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.delay.wheel; + +import java.util.Objects; + +/** + * @author xufeng.deng dennisdxf@gmail.com + * @since 2018-09-28 11:21 + */ +public class WheelLoadCursor { + private volatile int baseOffset; + private volatile long offset; + + private final Object cursorLock = new Object(); + + public static WheelLoadCursor create() { + return new WheelLoadCursor(); + } + + private WheelLoadCursor() { + this.baseOffset = -1; + this.offset = -1L; + } + + boolean shiftCursor(int shiftBaseOffset, long shiftOffset) { + if (shiftBaseOffset >= baseOffset) { + synchronized (cursorLock) { + this.baseOffset = shiftBaseOffset; + this.offset = shiftOffset; + } + return true; + } + + return false; + } + + boolean shiftCursor(int cursor) { + if (cursor >= baseOffset) { + synchronized (cursorLock) { + this.baseOffset = cursor; + this.offset = -1; + } + return true; + } + + return false; + } + + boolean shiftOffset(long loadedOffset) { + if (offset < loadedOffset) { + synchronized (cursorLock) { + offset = loadedOffset; + return true; + } + } + + return false; + } + + Cursor cursor() { + synchronized (cursorLock) { + return new Cursor(baseOffset, offset); + } + } + + public int baseOffset() { + return baseOffset; + } + + public long offset() { + synchronized (cursorLock) { + return offset; + } + } + + public static class Cursor { + private final int baseOffset; + private final long offset; + + public Cursor(int baseOffset, long offset) { + this.baseOffset = baseOffset; + this.offset = offset; + } + + public int getBaseOffset() { + return baseOffset; + } + + public long getOffset() { + return offset; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + Cursor cursor = (Cursor) o; + return baseOffset == cursor.baseOffset && + offset == cursor.offset; + } + + @Override + public int hashCode() { + + return Objects.hash(baseOffset, offset); + } + } +} diff --git a/qmq-delay-server/src/main/java/qunar/tc/qmq/delay/wheel/WheelTickManager.java b/qmq-delay-server/src/main/java/qunar/tc/qmq/delay/wheel/WheelTickManager.java new file mode 100644 index 00000000..5d5ea1a4 --- /dev/null +++ b/qmq-delay-server/src/main/java/qunar/tc/qmq/delay/wheel/WheelTickManager.java @@ -0,0 +1,309 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.delay.wheel; + +import com.google.common.base.Throwables; +import com.google.common.util.concurrent.ThreadFactoryBuilder; +import io.netty.buffer.ByteBuf; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import qunar.tc.qmq.broker.BrokerService; +import qunar.tc.qmq.configuration.DynamicConfigLoader; +import qunar.tc.qmq.delay.DelayLogFacade; +import qunar.tc.qmq.delay.ScheduleIndex; +import qunar.tc.qmq.delay.Switchable; +import qunar.tc.qmq.delay.base.LongHashSet; +import qunar.tc.qmq.delay.config.DefaultStoreConfiguration; +import qunar.tc.qmq.delay.config.StoreConfiguration; +import qunar.tc.qmq.delay.monitor.QMon; +import qunar.tc.qmq.delay.sender.DelayProcessor; +import qunar.tc.qmq.delay.sender.Sender; +import qunar.tc.qmq.delay.sender.SenderProcessor; +import qunar.tc.qmq.delay.store.log.DispatchLogSegment; +import qunar.tc.qmq.delay.store.log.ScheduleSetSegment; +import qunar.tc.qmq.delay.store.visitor.LogVisitor; +import qunar.tc.qmq.metrics.Metrics; + +import java.util.Optional; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; + +import static qunar.tc.qmq.delay.store.log.ScheduleOffsetResolver.resolveSegment; + +/** + * @author xufeng.deng dennisdxf@gmail.com + * @since 2018-07-19 20:19 + */ +public class WheelTickManager implements Switchable, HashedWheelTimer.Processor { + private static final Logger LOGGER = LoggerFactory.getLogger(WheelTickManager.class); + + private static final int TICKS_PER_WHEEL = 2 * 60 * 60; + + private final ScheduledExecutorService loadScheduler; + private final StoreConfiguration config; + private final DelayLogFacade facade; + private final HashedWheelTimer timer; + private final DelayProcessor sender; + private final AtomicBoolean started; + private final WheelLoadCursor loadingCursor; + private final WheelLoadCursor loadedCursor; + + public WheelTickManager(DefaultStoreConfiguration config, BrokerService brokerService, DelayLogFacade facade, Sender sender) { + this.config = config; + this.timer = new HashedWheelTimer(new ThreadFactoryBuilder().setNameFormat("delay-send-%d").build(), 500, TimeUnit.MILLISECONDS, TICKS_PER_WHEEL, this); + this.facade = facade; + this.sender = new SenderProcessor(facade, brokerService, sender, config.getConfig()); + this.started = new AtomicBoolean(false); + this.loadingCursor = WheelLoadCursor.create(); + this.loadedCursor = WheelLoadCursor.create(); + + this.loadScheduler = Executors.newSingleThreadScheduledExecutor(new ThreadFactoryBuilder().setNameFormat("wheel-segment-loader-%d").build()); + } + + @Override + public void start() { + if (!isStarted()) { + sender.init(); + timer.start(); + started.set(true); + recover(); + loadScheduler.scheduleWithFixedDelay(this::load, 0, config.getLoadSegmentDelayMinutes(), TimeUnit.MINUTES); + LOGGER.info("wheel started."); + } + } + + private void recover() { + LOGGER.info("wheel recover..."); + DispatchLogSegment currentDispatchedSegment = facade.latestDispatchSegment(); + if (currentDispatchedSegment == null) { + LOGGER.warn("load latest dispatch segment null"); + return; + } + + int latestOffset = currentDispatchedSegment.getSegmentBaseOffset(); + DispatchLogSegment lastSegment = facade.lowerDispatchSegment(latestOffset); + if (null != lastSegment) doRecover(lastSegment); + + doRecover(currentDispatchedSegment); + LOGGER.info("wheel recover done. currentOffset:{}", latestOffset); + } + + private void doRecover(DispatchLogSegment dispatchLogSegment) { + int segmentBaseOffset = dispatchLogSegment.getSegmentBaseOffset(); + ScheduleSetSegment setSegment = facade.loadScheduleLogSegment(segmentBaseOffset); + if (setSegment == null) { + LOGGER.error("load schedule index error,dispatch segment:{}", segmentBaseOffset); + return; + } + + LongHashSet dispatchedSet = loadDispatchLog(dispatchLogSegment); + WheelLoadCursor.Cursor loadCursor = facade.loadUnDispatch(setSegment, dispatchedSet, this::refresh); + int baseOffset = loadCursor.getBaseOffset(); + loadingCursor.shiftCursor(baseOffset, loadCursor.getOffset()); + loadedCursor.shiftCursor(baseOffset); + } + + private LongHashSet loadDispatchLog(final DispatchLogSegment currentDispatchLog) { + LogVisitor visitor = currentDispatchLog.newVisitor(0); + final LongHashSet recordSet = new LongHashSet(currentDispatchLog.entries()); + try { + while (true) { + Optional recordOptional = visitor.nextRecord(); + if (!recordOptional.isPresent()) break; + recordSet.set(recordOptional.get()); + } + return recordSet; + } finally { + visitor.close(); + } + } + + private boolean isStarted() { + return started.get(); + } + + private void load() { + long next = System.currentTimeMillis() + config.getLoadInAdvanceTimesInMillis(); + int prepareLoadBaseOffset = resolveSegment(next); + try { + loadUntil(prepareLoadBaseOffset); + } catch (InterruptedException ignored) { + LOGGER.debug("load segment interrupted"); + } + } + + private void loadUntil(int until) throws InterruptedException { + int loadedBaseOffset = loadedCursor.baseOffset(); + // have loaded + if (loadedBaseOffset > until) return; + + do { + // wait next turn when loaded error. + if (!loadUntilInternal(until)) break; + + // load successfully(no error happened) and current wheel loading cursor < until + if (loadingCursor.baseOffset() < until) { + long thresholdTime = System.currentTimeMillis() + config.getLoadBlockingExitTimesInMillis(); + // exit in a few minutes in advance + if (resolveSegment(thresholdTime) >= until) { + loadingCursor.shiftCursor(until); + loadedCursor.shiftCursor(until); + break; + } + } + + Thread.sleep(100); + } while (loadedCursor.baseOffset() < until); + + LOGGER.info("wheel load until {} <= {}", loadedCursor.baseOffset(), until); + } + + private boolean loadUntilInternal(int until) { + int index = resolveStartIndex(); + if (index < 0) return true; + + try { + while (index <= until) { + ScheduleSetSegment segment = facade.loadScheduleLogSegment(index); + if (segment == null) { + int nextIndex = facade.higherScheduleBaseOffset(index); + if (nextIndex < 0) return true; + index = nextIndex; + continue; + } + + loadSegment(segment); + int nextIndex = facade.higherScheduleBaseOffset(index); + if (nextIndex < 0) return true; + + index = nextIndex; + } + } catch (Throwable e) { + LOGGER.error("wheel load segment failed,currentSegmentOffset:{} until:{}", loadedCursor.baseOffset(), until, e); + QMon.loadSegmentFailed(); + return false; + } + + return true; + } + + /** + * resolve wheel-load start index + * + * @return generally, result > 0, however the result might be -1. -1 mean that no higher key. + */ + private int resolveStartIndex() { + WheelLoadCursor.Cursor loadedEntry = loadedCursor.cursor(); + int startIndex = loadedEntry.getBaseOffset(); + long offset = loadedEntry.getOffset(); + + if (offset < 0) return facade.higherScheduleBaseOffset(startIndex); + + return startIndex; + } + + private void loadSegment(ScheduleSetSegment segment) { + final long start = System.currentTimeMillis(); + try { + int baseOffset = segment.getSegmentBaseOffset(); + long offset = segment.getWrotePosition(); + if (!loadingCursor.shiftCursor(baseOffset, offset)) { + LOGGER.error("doLoadSegment error,shift loadingCursor failed,from {}-{} to {}-{}", loadingCursor.baseOffset(), loadingCursor.offset(), baseOffset, offset); + return; + } + + WheelLoadCursor.Cursor loadedCursorEntry = loadedCursor.cursor(); + // have loaded + if (baseOffset < loadedCursorEntry.getBaseOffset()) return; + + long startOffset = 0; + // last load action happened error + if (baseOffset == loadedCursorEntry.getBaseOffset() && loadedCursorEntry.getOffset() > -1) + startOffset = loadedCursorEntry.getOffset(); + + LogVisitor visitor = segment.newVisitor(startOffset, config.getSingleMessageLimitSize()); + try { + loadedCursor.shiftCursor(baseOffset, startOffset); + + long currentOffset = startOffset; + while (currentOffset < offset) { + Optional recordOptional = visitor.nextRecord(); + if (!recordOptional.isPresent()) break; + ByteBuf index = recordOptional.get(); + currentOffset = ScheduleIndex.offset(index) + ScheduleIndex.size(index); + refresh(index); + loadedCursor.shiftOffset(currentOffset); + } + loadedCursor.shiftCursor(baseOffset); + LOGGER.info("loaded segment:{} {}", loadedCursor.baseOffset(), currentOffset); + } finally { + visitor.close(); + } + } finally { + Metrics.timer("loadSegmentTimer").update(System.currentTimeMillis() - start, TimeUnit.MILLISECONDS); + } + } + + private void refresh(ByteBuf record) { + long now = System.currentTimeMillis(); + long scheduleTime = now; + try { + scheduleTime = ScheduleIndex.scheduleTime(record); + timer.newTimeout(record, scheduleTime - now, TimeUnit.MILLISECONDS); + } catch (Throwable e) { + LOGGER.error("wheel refresh error, scheduleTime:{}, delay:{}", scheduleTime, scheduleTime - now); + throw Throwables.propagate(e); + } + } + + @Override + public void shutdown() { + if (isStarted()) { + loadScheduler.shutdown(); + timer.stop(); + started.set(false); + sender.destroy(); + LOGGER.info("wheel shutdown."); + } + } + + public void addWHeel(ByteBuf record) { + refresh(record); + } + + public boolean canAdd(long scheduleTime, long offset) { + WheelLoadCursor.Cursor currentCursor = loadingCursor.cursor(); + int currentBaseOffset = currentCursor.getBaseOffset(); + long currentOffset = currentCursor.getOffset(); + + int baseOffset = resolveSegment(scheduleTime); + if (baseOffset < currentBaseOffset) return true; + + if (baseOffset == currentBaseOffset) { + return currentOffset <= offset; + } + return false; + } + + @Override + public void process(ByteBuf record) { + QMon.scheduleDispatch(); + sender.send(record); + } +} diff --git a/qmq-dist/assembly/bin.xml b/qmq-dist/assembly/bin.xml new file mode 100644 index 00000000..492cadd2 --- /dev/null +++ b/qmq-dist/assembly/bin.xml @@ -0,0 +1,35 @@ + + + + bin + + qmq-dist-${project.version}-bin + + + dir + tar.gz + + + + + + conf/** + + + + + + bin/** + + 0755 + + + + + + lib + false + false + + + \ No newline at end of file diff --git a/qmq-dist/bin/base.sh b/qmq-dist/bin/base.sh new file mode 100755 index 00000000..f1d09460 --- /dev/null +++ b/qmq-dist/bin/base.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash +set -euo pipefail + +QMQ_CFG_DIR="$QMQ_BIN_DIR/../conf" +QMQ_PID_DIR="$QMQ_BIN_DIR/../pid" +QMQ_LOG_DIR="$QMQ_BIN_DIR/../logs" + +if [[ ! -w "$QMQ_PID_DIR" ]] ; then +mkdir -p "$QMQ_PID_DIR" +fi + +if [[ ! -w "$QMQ_LOG_DIR" ]] ; then +mkdir -p "$QMQ_LOG_DIR" +fi + +CLASSPATH="$QMQ_CFG_DIR" +for i in "$QMQ_BIN_DIR"/../lib/* +do +CLASSPATH="$i:$CLASSPATH" +done diff --git a/qmq-dist/bin/broker-env.sh b/qmq-dist/bin/broker-env.sh new file mode 100644 index 00000000..d0580a77 --- /dev/null +++ b/qmq-dist/bin/broker-env.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash +set -euo pipefail + +JAVA_HOME="" +JAVA_OPTS="" diff --git a/qmq-dist/bin/broker.sh b/qmq-dist/bin/broker.sh new file mode 100644 index 00000000..24267492 --- /dev/null +++ b/qmq-dist/bin/broker.sh @@ -0,0 +1,67 @@ +#!/usr/bin/env bash +set -euo pipefail + +QMQ_BIN="${BASH_SOURCE-$0}" +QMQ_BIN="$(dirname "$QMQ_BIN")" +QMQ_BIN_DIR="$(cd "$QMQ_BIN"; pwd)" +QMQ_BROKER_MAIN="qunar.tc.qmq.container.Bootstrap" + +. "$QMQ_BIN_DIR/base.sh" +. "$QMQ_BIN_DIR/broker-env.sh" + +if [[ "$JAVA_HOME" != "" ]]; then + JAVA="$JAVA_HOME/bin/java" +else + JAVA=java +fi + +JAVA_OPTS="$JAVA_OPTS -DQMQ_LOG_DIR=$QMQ_LOG_DIR" +QMQ_PID_FILE="$QMQ_PID_DIR/broker.pid" +QMQ_DAEMON_OUT="$QMQ_LOG_DIR/broker.out" + +CMD=${1:-} +case ${CMD} in +start) + echo -n "Starting qmq broker ... " + if [[ -f "$QMQ_PID_FILE" ]]; then + if kill -0 `cat "$QMQ_PID_FILE"` > /dev/null 2>&1; then + echo already running as process `cat "$QMQ_PID_FILE"`. + exit 0 + fi + fi + nohup "$JAVA" -cp "$CLASSPATH" ${JAVA_OPTS} ${QMQ_BROKER_MAIN} > "$QMQ_DAEMON_OUT" 2>&1 < /dev/null & + if [[ $? -eq 0 ]] + then + /bin/echo -n $! > "$QMQ_PID_FILE" + if [[ $? -eq 0 ]]; + then + sleep 1 + echo STARTED + else + echo FAILED TO WRITE PID + exit 1 + fi + else + echo SERVER DID NOT START + exit 1 + fi + ;; +start-foreground) + ZOO_CMD=(exec "$JAVA") + "${ZOO_CMD[@]}" -cp "$CLASSPATH" ${JAVA_OPTS} ${QMQ_BROKER_MAIN} + ;; +stop) + echo -n "Stopping qmq broker ... " + if [[ ! -f "$QMQ_PID_FILE" ]] + then + echo "no broker to stop (could not find file $QMQ_PID_FILE)" + else + kill -9 $(cat "$QMQ_PID_FILE") + rm "$QMQ_PID_FILE" + echo STOPPED + fi + exit 0 + ;; +*) + echo "Usage: $0 {start|start-foreground|stop}" >&2 +esac \ No newline at end of file diff --git a/qmq-dist/bin/delay-env.sh b/qmq-dist/bin/delay-env.sh new file mode 100755 index 00000000..d0580a77 --- /dev/null +++ b/qmq-dist/bin/delay-env.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash +set -euo pipefail + +JAVA_HOME="" +JAVA_OPTS="" diff --git a/qmq-dist/bin/delay.sh b/qmq-dist/bin/delay.sh new file mode 100755 index 00000000..ae7cf8d7 --- /dev/null +++ b/qmq-dist/bin/delay.sh @@ -0,0 +1,67 @@ +#!/usr/bin/env bash +set -euo pipefail + +QMQ_BIN="${BASH_SOURCE-$0}" +QMQ_BIN="$(dirname "$QMQ_BIN")" +QMQ_BIN_DIR="$(cd "$QMQ_BIN"; pwd)" +QMQ_DELAY_MAIN="qunar.tc.qmq.delay.container.Bootstrap" + +. "$QMQ_BIN_DIR/base.sh" +. "$QMQ_BIN_DIR/delay-env.sh" + +if [[ "$JAVA_HOME" != "" ]]; then + JAVA="$JAVA_HOME/bin/java" +else + JAVA=java +fi + +JAVA_OPTS="${JAVA_OPTS} -DQMQ_LOG_DIR=${QMQ_LOG_DIR}" +QMQ_PID_FILE="$QMQ_PID_DIR/delay.pid" +QMQ_DAEMON_OUT="$QMQ_LOG_DIR/delay.out" + +CMD=${1:-} +case ${CMD} in +start) + echo -n "Starting qmq delay server ... " + if [[ -f "$QMQ_PID_FILE" ]]; then + if kill -0 `cat "$QMQ_PID_FILE"` > /dev/null 2>&1; then + echo already running as process `cat "$QMQ_PID_FILE"`. + exit 0 + fi + fi + nohup "$JAVA" -cp "$CLASSPATH" ${JAVA_OPTS} ${QMQ_DELAY_MAIN} > "$QMQ_DAEMON_OUT" 2>&1 < /dev/null & + if [[ $? -eq 0 ]] + then + /bin/echo -n $! > "$QMQ_PID_FILE" + if [[ $? -eq 0 ]]; + then + sleep 1 + echo STARTED + else + echo FAILED TO WRITE PID + exit 1 + fi + else + echo SERVER DID NOT START + exit 1 + fi + ;; +start-foreground) + ZOO_CMD=(exec "$JAVA") + "${ZOO_CMD[@]}" -cp "$CLASSPATH" ${JAVA_OPTS} ${QMQ_DELAY_MAIN} + ;; +stop) + echo -n "Stopping qmq delay server ... " + if [[ ! -f "$QMQ_PID_FILE" ]] + then + echo "no delay server to stop (could not find file $QMQ_PID_FILE)" + else + kill -9 $(cat "$QMQ_PID_FILE") + rm "$QMQ_PID_FILE" + echo STOPPED + fi + exit 0 + ;; +*) + echo "Usage: $0 {start|start-foreground|stop}" >&2 +esac diff --git a/qmq-dist/bin/metaserver-env.sh b/qmq-dist/bin/metaserver-env.sh new file mode 100755 index 00000000..ec25d381 --- /dev/null +++ b/qmq-dist/bin/metaserver-env.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash +set -euo pipefail + +JAVA_HOME="" +JAVA_OPTS="-Xms1g -Xmx1g" diff --git a/qmq-dist/bin/metaserver.sh b/qmq-dist/bin/metaserver.sh new file mode 100755 index 00000000..5f7101c5 --- /dev/null +++ b/qmq-dist/bin/metaserver.sh @@ -0,0 +1,67 @@ +#!/usr/bin/env bash +set -euo pipefail + +QMQ_BIN="${BASH_SOURCE-$0}" +QMQ_BIN="$(dirname "$QMQ_BIN")" +QMQ_BIN_DIR="$(cd "$QMQ_BIN"; pwd)" +QMQ_META_MAIN="qunar.tc.qmq.meta.startup.Bootstrap" + +. "$QMQ_BIN_DIR/base.sh" +. "$QMQ_BIN_DIR/metaserver-env.sh" + +if [[ "$JAVA_HOME" != "" ]]; then + JAVA="$JAVA_HOME/bin/java" +else + JAVA=java +fi + +JAVA_OPTS="$JAVA_OPTS -DQMQ_LOG_DIR=$QMQ_LOG_DIR" +QMQ_PID_FILE="$QMQ_PID_DIR/metaserver.pid" +QMQ_DAEMON_OUT="$QMQ_LOG_DIR/metaserver.out" + +CMD=${1:-} +case ${CMD} in +start) + echo -n "Starting qmq meta server ... " + if [[ -f "$QMQ_PID_FILE" ]]; then + if kill -0 `cat "$QMQ_PID_FILE"` > /dev/null 2>&1; then + echo already running as process `cat "$QMQ_PID_FILE"`. + exit 0 + fi + fi + nohup "$JAVA" -cp "$CLASSPATH" ${JAVA_OPTS} ${QMQ_META_MAIN} > "$QMQ_DAEMON_OUT" 2>&1 < /dev/null & + if [[ $? -eq 0 ]] + then + /bin/echo -n $! > "$QMQ_PID_FILE" + if [[ $? -eq 0 ]]; + then + sleep 1 + echo STARTED + else + echo FAILED TO WRITE PID + exit 1 + fi + else + echo SERVER DID NOT START + exit 1 + fi + ;; +start-foreground) + ZOO_CMD=(exec "$JAVA") + "${ZOO_CMD[@]}" -cp "$CLASSPATH" ${JAVA_OPTS} ${QMQ_META_MAIN} + ;; +stop) + echo -n "Stopping qmq meta server ... " + if [[ ! -f "$QMQ_PID_FILE" ]] + then + echo "no meta server to stop (could not find file $QMQ_PID_FILE)" + else + kill -9 $(cat "$QMQ_PID_FILE") + rm "$QMQ_PID_FILE" + echo STOPPED + fi + exit 0 + ;; +*) + echo "Usage: $0 {start|start-foreground|stop}" >&2 +esac \ No newline at end of file diff --git a/qmq-dist/bin/tools-env.sh b/qmq-dist/bin/tools-env.sh new file mode 100755 index 00000000..b174370a --- /dev/null +++ b/qmq-dist/bin/tools-env.sh @@ -0,0 +1,6 @@ +#!/usr/bin/env bash +set -euo pipefail + +JAVA_HOME="" +JAVA_OPTS="-Xms256m -Xmx256m" + diff --git a/qmq-dist/bin/tools.sh b/qmq-dist/bin/tools.sh new file mode 100755 index 00000000..f277a07a --- /dev/null +++ b/qmq-dist/bin/tools.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash +set -euo pipefail + +QMQ_BIN="${BASH_SOURCE-$0}" +QMQ_BIN="$(dirname "${QMQ_BIN}")" +QMQ_BIN_DIR="$(cd "${QMQ_BIN}"; pwd)" +QMQ_TOOLS_MAIN="qunar.tc.qmq.tools.Tools" + +. "$QMQ_BIN_DIR/base.sh" +. "$QMQ_BIN_DIR/tools-env.sh" + +if [[ "$JAVA_HOME" != "" ]]; then + JAVA="$JAVA_HOME/bin/java" +else + JAVA=java +fi + +ZOO_CMD=(exec "$JAVA") +"${ZOO_CMD[@]}" -cp "$CLASSPATH" ${JAVA_OPTS} ${QMQ_TOOLS_MAIN} $@ diff --git a/qmq-dist/conf/broker.properties b/qmq-dist/conf/broker.properties new file mode 100644 index 00000000..aec9c7d1 --- /dev/null +++ b/qmq-dist/conf/broker.properties @@ -0,0 +1,4 @@ +store.root=/data +meta.server.endpoint=http://[:]/meta/address +messagelog.retention.hours=1 +log.expired.delete.enable=true \ No newline at end of file diff --git a/qmq-dist/conf/datasource.properties b/qmq-dist/conf/datasource.properties new file mode 100644 index 00000000..36e24209 --- /dev/null +++ b/qmq-dist/conf/datasource.properties @@ -0,0 +1,5 @@ +jdbc.driverClassName=com.mysql.jdbc.Driver +jdbc.url=jdbc:mysql://:/ +jdbc.username= +jdbc.password= +pool.size.max=10 \ No newline at end of file diff --git a/qmq-dist/conf/delay.properties b/qmq-dist/conf/delay.properties new file mode 100644 index 00000000..c7c75a79 --- /dev/null +++ b/qmq-dist/conf/delay.properties @@ -0,0 +1,4 @@ +meta.server.endpoint=http://[:]/meta/address +broker.port=20801 +sync.port=20802 +store.root=/data diff --git a/qmq-dist/conf/logback.xml b/qmq-dist/conf/logback.xml new file mode 100644 index 00000000..efd07666 --- /dev/null +++ b/qmq-dist/conf/logback.xml @@ -0,0 +1,14 @@ + + + + + + + [%d{yyyy-MM-dd HH:mm:ss} %5p %c] %m%n + + + + + + + \ No newline at end of file diff --git a/qmq-dist/conf/metaserver.properties b/qmq-dist/conf/metaserver.properties new file mode 100644 index 00000000..566300e4 --- /dev/null +++ b/qmq-dist/conf/metaserver.properties @@ -0,0 +1 @@ +min.group.num=2 \ No newline at end of file diff --git a/qmq-dist/conf/valid-api-tokens.properties b/qmq-dist/conf/valid-api-tokens.properties new file mode 100644 index 00000000..e69de29b diff --git a/qmq-dist/pom.xml b/qmq-dist/pom.xml new file mode 100644 index 00000000..57fb57b6 --- /dev/null +++ b/qmq-dist/pom.xml @@ -0,0 +1,58 @@ + + + 4.0.0 + + + qmq + qunar.tc + 4.0.30 + + + qmq-dist + pom + + + + qunar.tc + qmq-metaserver + + + qunar.tc + qmq-server + + + qunar.tc + qmq-delay-server + + + qunar.tc + qmq-tools + + + + + + + maven-assembly-plugin + + + assemble + package + + single + + + qmq-dist-${project.version} + + assembly/bin.xml + + gnu + + + + + + + \ No newline at end of file diff --git a/qmq-metaserver/pom.xml b/qmq-metaserver/pom.xml new file mode 100644 index 00000000..c8902ad5 --- /dev/null +++ b/qmq-metaserver/pom.xml @@ -0,0 +1,100 @@ + + + 4.0.0 + + + qmq + qunar.tc + 4.0.30 + + + qmq-metaserver + jar + + + + ${project.groupId} + qmq-common + + + ${project.groupId} + qmq-remoting + + + ${project.groupId} + qmq-client + + + qunar.tc + qmq-store + + + ${project.groupId} + qmq-server-common + + + + com.google.guava + guava + + + org.eclipse.jetty + jetty-server + + + org.eclipse.jetty + jetty-servlet + + + javax.servlet + javax.servlet-api + + + org.slf4j + slf4j-api + + + org.slf4j + jcl-over-slf4j + + + org.slf4j + log4j-over-slf4j + + + ch.qos.logback + logback-classic + + + ch.qos.logback + logback-core + + + + org.springframework + spring-jdbc + + + com.zaxxer + HikariCP + + + mysql + mysql-connector-java + + + com.fasterxml.jackson.core + jackson-core + + + com.fasterxml.jackson.core + jackson-databind + + + + + ROOT + + \ No newline at end of file diff --git a/qmq-metaserver/sql/init.sql b/qmq-metaserver/sql/init.sql new file mode 100644 index 00000000..c0c7026c --- /dev/null +++ b/qmq-metaserver/sql/init.sql @@ -0,0 +1,96 @@ +CREATE TABLE `broker_group` +( + `id` INT(11) UNSIGNED NOT NULL AUTO_INCREMENT COMMENT '主键', + `group_name` VARCHAR(30) NOT NULL DEFAULT '' COMMENT 'group名字', + `kind` INT NOT NULL DEFAULT '-1' COMMENT '类型', + `master_address` VARCHAR(25) NOT NULL DEFAULT '' COMMENT 'master地址,ip:port', + `broker_state` TINYINT NOT NULL DEFAULT '-1' COMMENT 'broker master 状态', + `tag` VARCHAR(30) NOT NULL DEFAULT '' COMMENT '分组的标签', + `create_time` TIMESTAMP NOT NULL DEFAULT '1970-01-01 08:00:01' COMMENT '创建时间', + `update_time` TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '修改时间', + PRIMARY KEY (`id`), + UNIQUE KEY `uniq_group_name` (`group_name`) +) ENGINE = InnoDB + DEFAULT CHARSET = utf8mb4 COMMENT = 'broker组信息'; + +CREATE TABLE `subject_info` +( + `id` INT(11) UNSIGNED NOT NULL AUTO_INCREMENT COMMENT '主键', + `name` VARCHAR(100) NOT NULL DEFAULT '' COMMENT '主题名', + `tag` VARCHAR(30) NOT NULL DEFAULT '' COMMENT 'tag', + `create_time` TIMESTAMP NOT NULL DEFAULT '1970-01-01 08:00:01' COMMENT '创建时间', + `update_time` TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '修改时间', + PRIMARY KEY (`id`), + UNIQUE KEY `uniq_name` (`name`) +) ENGINE = InnoDB + DEFAULT CHARSET = utf8mb4 + COMMENT = '主题信息'; + +CREATE TABLE `subject_route` +( + `id` INT(11) UNSIGNED NOT NULL AUTO_INCREMENT COMMENT '主键', + `subject_info` VARCHAR(100) NOT NULL DEFAULT '' COMMENT '主题', + `broker_group_json` VARCHAR(300) NOT NULL DEFAULT '' COMMENT 'broker group name信息,json存储', + `version` INT NOT NULL DEFAULT 0 COMMENT '版本信息', + `create_time` TIMESTAMP NOT NULL DEFAULT '1970-01-01 08:00:01' + COMMENT '创建时间', + `update_time` TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP + COMMENT '修改时间', + PRIMARY KEY (`id`), + UNIQUE KEY `uniq_subject` (`subject_info`) +) + ENGINE = InnoDB + DEFAULT CHARSET = utf8mb4 + COMMENT = '主题路由信息'; + +CREATE TABLE client_meta_info +( + `id` INT(11) UNSIGNED NOT NULL AUTO_INCREMENT COMMENT '主键', + `subject_info` VARCHAR(100) NOT NULL DEFAULT '' COMMENT '主题', + `client_type` TINYINT NOT NULL DEFAULT 0 COMMENT 'client类型', + `consumer_group` VARCHAR(100) NOT NULL DEFAULT '' COMMENT '消费组', + `client_id` VARCHAR(100) NOT NULL DEFAULT '' COMMENT 'client id', + `app_code` VARCHAR(100) NOT NULL DEFAULT '' COMMENT '应用', + `room` VARCHAR(20) NOT NULL DEFAULT '' COMMENT '机房', + `create_time` TIMESTAMP NOT NULL DEFAULT '1970-01-01 08:00:01' COMMENT '创建时间', + `update_time` TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '修改时间', + PRIMARY KEY (`id`), + UNIQUE KEY `uniq_subject_client_type_consumer_group_client_id` (`subject_info`, `client_type`, `consumer_group`, `client_id`) +) + ENGINE = InnoDB + DEFAULT CHARSET = utf8mb4 + COMMENT = '订阅关系表'; + +CREATE TABLE `client_offline_state` +( + `id` INT(11) UNSIGNED NOT NULL AUTO_INCREMENT COMMENT '主键', + `client_id` VARCHAR(100) NOT NULL DEFAULT '' COMMENT 'client id', + `subject_info` VARCHAR(100) NOT NULL DEFAULT '' COMMENT '主题', + `consumer_group` VARCHAR(100) NOT NULL DEFAULT '' COMMENT '消费组', + `state` TINYINT NOT NULL DEFAULT 0 COMMENT '上下线状态,0上线,1下线', + PRIMARY KEY (`id`), + UNIQUE KEY `uniq_client_id_subject_group` (`client_id`, `subject_info`, `consumer_group`) +) + ENGINE = InnoDB + DEFAULT CHARSET = utf8mb4 + COMMENT = '下线的client'; + +CREATE TABLE `broker` +( + `id` BIGINT UNSIGNED NOT NULL AUTO_INCREMENT COMMENT '自增主键', + `group_name` VARCHAR(30) NOT NULL DEFAULT '' COMMENT 'group名字', + `hostname` VARCHAR(50) NOT NULL DEFAULT '' COMMENT '机器名', + `ip` INT UNSIGNED NOT NULL DEFAULT '0' COMMENT '机器IP', + `role` INT NOT NULL DEFAULT '-1' COMMENT '角色', + `serve_port` INT NOT NULL DEFAULT '20881' COMMENT '客户端请求端口', + `sync_port` INT NOT NULL DEFAULT '20882' COMMENT '从机同步端口', + `create_time` TIMESTAMP NOT NULL DEFAULT '2018-01-01 01:01:01' COMMENT '创建时间', + `update_time` TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '修改时间', + + PRIMARY KEY (`id`), + UNIQUE KEY uniq_broker (`hostname`, `serve_port`), + UNIQUE KEY uniq_group_role (`group_name`, `role`), + KEY idx_broker_group (`group_name`) +) ENGINE InnoDB + DEFAULT CHARSET = utf8mb4 + COMMENT 'broker列表'; diff --git a/qmq-metaserver/src/main/java/qunar/tc/qmq/meta/cache/AliveClientManager.java b/qmq-metaserver/src/main/java/qunar/tc/qmq/meta/cache/AliveClientManager.java new file mode 100644 index 00000000..5071dc89 --- /dev/null +++ b/qmq-metaserver/src/main/java/qunar/tc/qmq/meta/cache/AliveClientManager.java @@ -0,0 +1,151 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.meta.cache; + +import com.google.common.collect.Sets; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import qunar.tc.qmq.concurrent.NamedThreadFactory; +import qunar.tc.qmq.meta.model.ClientMetaInfo; +import qunar.tc.qmq.meta.monitor.QMon; +import qunar.tc.qmq.protocol.consumer.MetaInfoRequest; + +import java.util.Collections; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; + +/** + * @author yunfeng.yang + * @since 2018/1/2 + */ +public class AliveClientManager { + private static final Logger LOG = LoggerFactory.getLogger(AliveClientManager.class); + + private static final ScheduledExecutorService EXECUTOR = Executors.newSingleThreadScheduledExecutor(new NamedThreadFactory("client-info-cleaner-")); + + private static final long EXPIRE_TIME_MS = TimeUnit.SECONDS.toMillis(200L); + private static final long CLEAN_PERIOD_HOUR = 1L; + + private static final AliveClientManager INSTANCE = new AliveClientManager(); + + private final Map> allClients; + + private final Map> allSubject; + + private AliveClientManager() { + allClients = new ConcurrentHashMap<>(); + allSubject = new ConcurrentHashMap<>(); + startCleaner(); + } + + public static AliveClientManager getInstance() { + return INSTANCE; + } + + public void renew(final MetaInfoRequest request) { + try { + QMon.clientRefreshMetaInfoCountInc(request.getSubject()); + + final ClientMetaInfo meta = createClientMeta(request); + final Map subjectClients = allClients.computeIfAbsent(request.getSubject(), key -> new ConcurrentHashMap<>()); + subjectClients.put(meta, System.currentTimeMillis()); + final Map appCodeClients = allSubject.computeIfAbsent(request.getAppCode(), key -> new ConcurrentHashMap<>()); + appCodeClients.put(meta, System.currentTimeMillis()); + } catch (Exception e) { + LOG.error("refresh client info error", e); + } + } + + private ClientMetaInfo createClientMeta(MetaInfoRequest request) { + final ClientMetaInfo meta = new ClientMetaInfo(); + meta.setSubject(request.getSubject()); + meta.setConsumerGroup(request.getConsumerGroup()); + meta.setClientTypeCode(request.getClientTypeCode()); + meta.setAppCode(request.getAppCode()); + meta.setClientId(request.getClientId()); + return meta; + } + + public Set aliveClientsOf(String subject) { + final Map clients = allClients.get(subject); + if (clients == null || clients.isEmpty()) { + return Collections.emptySet(); + } + + final long now = System.currentTimeMillis(); + final Set result = new HashSet<>(); + for (Map.Entry entry : clients.entrySet()) { + if (entry.getValue() != null && now - entry.getValue() < EXPIRE_TIME_MS) { + result.add(entry.getKey()); + } + } + return result; + } + + public Set aliveSubjectByAppCode(String appCode) { + final Map clients = allSubject.get(appCode); + if (clients == null || clients.isEmpty()) { + return Collections.emptySet(); + } + final long now = System.currentTimeMillis(); + final Set result = Sets.newHashSet(); + for (Map.Entry entry : clients.entrySet()) { + if (entry.getValue() != null && now - entry.getValue() < EXPIRE_TIME_MS) { + result.add(entry.getKey()); + } + } + return result; + } + + private void startCleaner() { + EXECUTOR.scheduleAtFixedRate(new CleanTask(), CLEAN_PERIOD_HOUR, CLEAN_PERIOD_HOUR, TimeUnit.HOURS); + } + + private class CleanTask implements Runnable { + @Override + public void run() { + try { + removeExpiredClients(); + } catch (Exception e) { + LOG.error("clean dead client info failed.", e); + } + } + + private void removeExpiredClients() { + final long now = System.currentTimeMillis(); + for (Map map : allClients.values()) { + for (Map.Entry entry : map.entrySet()) { + if (now - entry.getValue() >= EXPIRE_TIME_MS) { + map.remove(entry.getKey()); + } + } + } + for (Map map : allSubject.values()) { + for (Map.Entry entry : map.entrySet()) { + if (now - entry.getValue() >= EXPIRE_TIME_MS) { + map.remove(entry.getKey()); + } + } + } + } + } +} \ No newline at end of file diff --git a/qmq-metaserver/src/main/java/qunar/tc/qmq/meta/cache/CachedMetaInfoManager.java b/qmq-metaserver/src/main/java/qunar/tc/qmq/meta/cache/CachedMetaInfoManager.java new file mode 100644 index 00000000..3320fe39 --- /dev/null +++ b/qmq-metaserver/src/main/java/qunar/tc/qmq/meta/cache/CachedMetaInfoManager.java @@ -0,0 +1,231 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.meta.cache; + +import com.google.common.base.Strings; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Lists; +import com.google.common.util.concurrent.ThreadFactoryBuilder; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import qunar.tc.qmq.common.Disposable; +import qunar.tc.qmq.configuration.DynamicConfig; +import qunar.tc.qmq.meta.BrokerGroup; +import qunar.tc.qmq.meta.BrokerGroupKind; +import qunar.tc.qmq.meta.model.SubjectInfo; +import qunar.tc.qmq.meta.model.SubjectRoute; +import qunar.tc.qmq.meta.store.Store; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; + +/** + * @author yunfeng.yang + * @since 2017/8/30 + */ +public class CachedMetaInfoManager implements Disposable { + private static final Logger LOG = LoggerFactory.getLogger(CachedMetaInfoManager.class); + + private static final ScheduledExecutorService SCHEDULE_POOL = Executors.newSingleThreadScheduledExecutor( + new ThreadFactoryBuilder().setNameFormat("meta-info-refresh-%d").build()); + private static final long DEFAULT_REFRESH_PERIOD_SECONDS = 5L; + + private static final String DEFAULT_BROKER_GROUP_TAG = "_default_"; + + private final Store store; + private final long refreshPeriodSeconds; + + /** + * subject -> groupNames + */ + private volatile Map> cachedSubjectGroups = new HashMap<>(); + /** + * groupName -> brokerGroup + */ + private volatile Map cachedBrokerGroups = new HashMap<>(); + /** + * tag -> groupNames + */ + private volatile Map> cachedTagToBrokerGroups = new HashMap<>(); + /** + * groupName -> brokerNewGroup + */ + private volatile Map cachedDelayNewGroups = new HashMap<>(); + /** + * subjectName -> subjectInfo + */ + private volatile Map cachedSubjectInfoMap = new HashMap<>(); + + public CachedMetaInfoManager(DynamicConfig config, Store store) { + this.refreshPeriodSeconds = config.getLong("refresh.period.seconds", DEFAULT_REFRESH_PERIOD_SECONDS); + this.store = store; + refresh(); + initRefreshTask(); + } + + public SubjectInfo getSubjectInfo(String subject) { + return cachedSubjectInfoMap.get(subject); + } + + private void initRefreshTask() { + SCHEDULE_POOL.scheduleAtFixedRate(new RefreshTask(), refreshPeriodSeconds, refreshPeriodSeconds, TimeUnit.SECONDS); + } + + public List getGroups(String subject) { + final List groups = cachedSubjectGroups.get(subject); + if (groups == null) { + return ImmutableList.of(); + } + return ImmutableList.copyOf(groups); + } + + public List getAllBrokerGroupNamesByTag(String tag) { + if (tag == null) { + return ImmutableList.of(); + } + List groupNames = cachedTagToBrokerGroups.get(tag); + return groupNames == null ? ImmutableList.of() : ImmutableList.copyOf(groupNames); + } + + public List getAllDefaultTagBrokerGroupNames() { + return ImmutableList.copyOf(cachedTagToBrokerGroups.get(DEFAULT_BROKER_GROUP_TAG)); + } + + public BrokerGroup getBrokerGroup(String groupName) { + return cachedBrokerGroups.get(groupName); + } + + public List getDelayNewGroups() { + return Lists.newArrayList(cachedDelayNewGroups.values()); + } + + public void executeRefreshTask() { + try { + SCHEDULE_POOL.execute(new RefreshTask()); + } catch (Exception e) { + LOG.error("execute refresh task reject"); + } + } + + private void refresh() { + LOG.info("refresh meta info"); + refreshBrokerGroups(); + refreshSubjectInfoCache(); + refreshGroupsAndSubjects(); + } + + private void refreshGroupsAndSubjects() { + final List subjectRoutes = store.getAllSubjectRoutes(); + if (subjectRoutes == null || subjectRoutes.size() == 0) { + return; + } + + Map aliveGroups = new HashMap<>(cachedBrokerGroups); + + final Map> groupSubjects = new HashMap<>(); + final Map> subjectGroups = new HashMap<>(); + for (SubjectRoute subjectRoute : subjectRoutes) { + final String subject = subjectRoute.getSubject(); + final List aliveGroupName = new ArrayList<>(); + + for (String groupName : subjectRoute.getBrokerGroups()) { + if (aliveGroups.containsKey(groupName)) { + List value = groupSubjects.computeIfAbsent(groupName, k -> new ArrayList<>()); + value.add(subject); + aliveGroupName.add(groupName); + } + } + subjectGroups.put(subject, aliveGroupName); + } + + cachedSubjectGroups = subjectGroups; + } + + private void refreshBrokerGroups() { + final List allBrokerGroups = store.getAllBrokerGroups(); + if (allBrokerGroups == null || allBrokerGroups.size() == 0) { + return; + } + + final Map normalBrokerGroups = new HashMap<>(); + final Map> tagToBrokerGroups = new HashMap<>(); + final Map delayNewGroups = new HashMap<>(); + + for (BrokerGroup brokerGroup : allBrokerGroups) { + String name = brokerGroup.getGroupName(); + if (Strings.isNullOrEmpty(name)) continue; + + if (brokerGroup.getKind() == BrokerGroupKind.DELAY) { + delayNewGroups.put(name, brokerGroup); + } else { + normalBrokerGroups.put(name, brokerGroup); + + String tag = brokerGroup.getTag(); + tag = Strings.isNullOrEmpty(tag) ? DEFAULT_BROKER_GROUP_TAG : tag; + List groups = tagToBrokerGroups.computeIfAbsent(tag, key -> new ArrayList<>()); + groups.add(name); + } + + } + + // ensure default tag has at least one empty list value + tagToBrokerGroups.computeIfAbsent(DEFAULT_BROKER_GROUP_TAG, key -> new ArrayList<>()); + + cachedBrokerGroups = normalBrokerGroups; + cachedTagToBrokerGroups = tagToBrokerGroups; + cachedDelayNewGroups = delayNewGroups; + } + + private void refreshSubjectInfoCache() { + final List subjectInfoList = store.getAllSubjectInfo(); + if (subjectInfoList == null || subjectInfoList.size() == 0) { + return; + } + final Map subjectInfoMap = new HashMap<>(); + for (SubjectInfo subjectInfo : subjectInfoList) { + String name = subjectInfo.getName(); + if (Strings.isNullOrEmpty(name)) { + continue; + } + + subjectInfoMap.put(name, subjectInfo); + } + cachedSubjectInfoMap = subjectInfoMap; + } + + @Override + public void destroy() { + SCHEDULE_POOL.shutdown(); + } + + private class RefreshTask implements Runnable { + + @Override + public void run() { + try { + refresh(); + } catch (Exception e) { + LOG.error("refresh meta info failed", e); + } + } + } +} diff --git a/qmq-metaserver/src/main/java/qunar/tc/qmq/meta/cache/CachedOfflineStateManager.java b/qmq-metaserver/src/main/java/qunar/tc/qmq/meta/cache/CachedOfflineStateManager.java new file mode 100644 index 00000000..374b5b42 --- /dev/null +++ b/qmq-metaserver/src/main/java/qunar/tc/qmq/meta/cache/CachedOfflineStateManager.java @@ -0,0 +1,135 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.meta.cache; + +import com.google.common.base.Suppliers; +import com.google.common.util.concurrent.ThreadFactoryBuilder; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import qunar.tc.qmq.base.OnOfflineState; +import qunar.tc.qmq.common.Disposable; +import qunar.tc.qmq.meta.model.ClientOfflineState; +import qunar.tc.qmq.meta.store.ClientOfflineStore; +import qunar.tc.qmq.meta.store.impl.ClientOfflineStoreImpl; +import qunar.tc.qmq.metrics.Metrics; +import qunar.tc.qmq.metrics.QmqCounter; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; +import java.util.function.Supplier; + +/** + * yiqun.fan@qunar.com 2018/3/2 + */ +public class CachedOfflineStateManager implements Disposable { + private static final Logger log = LoggerFactory.getLogger(CachedOfflineStateManager.class); + private static final long REFRESH_PERIOD_SECONDS = 30L; + private static final String CLIENT_STATE_KEY_SEPARATOR = "$"; + private static final String CLIENTID_OF_GROUP = "*"; + private static final QmqCounter REFRESH_ERROR = Metrics.counter("refresh_onofflinestate_error"); + + public static final Supplier SUPPLIER = Suppliers.memoize(CachedOfflineStateManager::new)::get; + + private final ClientOfflineStore store = new ClientOfflineStoreImpl(); + private final ScheduledExecutorService scheduledExecutor; + + private volatile long updateTime = -1; + private volatile Map groupStateMap = new HashMap<>(); + private volatile Map clientStateMap = new HashMap<>(); + + private static String clientStateKey(String clientId, String subject, String consumerGroup) { + return clientId + CLIENT_STATE_KEY_SEPARATOR + subject + CLIENT_STATE_KEY_SEPARATOR + consumerGroup; + } + + private CachedOfflineStateManager() { + refresh(); + scheduledExecutor = Executors.newSingleThreadScheduledExecutor(new ThreadFactoryBuilder().setNameFormat("meta-info-offlinestate-refresh-%d").build()); + scheduledExecutor.scheduleAtFixedRate(this::refresh, REFRESH_PERIOD_SECONDS, REFRESH_PERIOD_SECONDS, TimeUnit.SECONDS); + log.info("CachedOfflineStateManager started"); + } + + private void refresh() { + try { + long updateTime = store.now(); + List states = store.selectAll(); + Map groupStateMap = new HashMap<>(); + Map clientStateMap = new HashMap<>(); + for (ClientOfflineState state : states) { + if (state == null) continue; + + String key = clientStateKey(state.getClientId(), state.getSubject(), state.getConsumerGroup()); + if (CLIENTID_OF_GROUP.equals(state.getClientId())) { + groupStateMap.put(key, state.getState()); + } else { + clientStateMap.put(key, state.getState()); + } + } + this.updateTime = updateTime; + this.groupStateMap = groupStateMap; + this.clientStateMap = clientStateMap; + log.info("refreshed onoffline state {}", updateTime); + } catch (Exception e) { + log.error("refresh OfflineState exception", e); + REFRESH_ERROR.inc(); + } + } + + public long getLastUpdateTimestamp() { + return updateTime; + } + + public OnOfflineState queryClientState(String clientId, String subject, String consumerGroup) { + OnOfflineState state = groupStateMap.get(clientStateKey(CLIENTID_OF_GROUP, subject, consumerGroup)); + if (state != null) { + return state; + } + state = clientStateMap.get(clientStateKey(clientId, subject, consumerGroup)); + return state == null ? OnOfflineState.ONLINE : state; + } + + public void insertOrUpdate(ClientOfflineState clientState) { + if (clientState.getState() == OnOfflineState.OFFLINE) { + store.insertOrUpdate(clientState); + } else { + if (CLIENTID_OF_GROUP.equals(clientState.getClientId())) { + store.delete(clientState.getSubject(), clientState.getConsumerGroup()); + } else { + store.delete(clientState.getClientId(), clientState.getSubject(), clientState.getConsumerGroup()); + } + } + } + + public void queryClientState(ClientOfflineState clientState) { + Optional result = store.select(clientState.getClientId(), clientState.getSubject(), clientState.getConsumerGroup()); + if (result.isPresent()) { + clientState.setState(result.get().getState()); + } else { + clientState.setState(OnOfflineState.ONLINE); + } + } + + @Override + public void destroy() { + scheduledExecutor.shutdown(); + log.info("CachedOfflineStateManager destoried"); + } +} diff --git a/qmq-metaserver/src/main/java/qunar/tc/qmq/meta/container/WebContainerListener.java b/qmq-metaserver/src/main/java/qunar/tc/qmq/meta/container/WebContainerListener.java new file mode 100644 index 00000000..c3df8008 --- /dev/null +++ b/qmq-metaserver/src/main/java/qunar/tc/qmq/meta/container/WebContainerListener.java @@ -0,0 +1,43 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.meta.container; + + +import qunar.tc.qmq.meta.startup.ServerWrapper; + +import javax.servlet.ServletContextEvent; +import javax.servlet.ServletContextListener; + +/** + * User: zhaohuiyu Date: 1/7/13 Time: 4:35 PM + */ +public class WebContainerListener implements ServletContextListener { + private ServerWrapper wrapper; + + @Override + public void contextInitialized(ServletContextEvent sce) { + wrapper = new ServerWrapper(); + wrapper.start(sce.getServletContext()); + } + + @Override + public void contextDestroyed(ServletContextEvent sce) { + if (this.wrapper != null) { + wrapper.destroy(); + } + } +} diff --git a/qmq-metaserver/src/main/java/qunar/tc/qmq/meta/loadbalance/AbstractLoadBalance.java b/qmq-metaserver/src/main/java/qunar/tc/qmq/meta/loadbalance/AbstractLoadBalance.java new file mode 100644 index 00000000..acfd0796 --- /dev/null +++ b/qmq-metaserver/src/main/java/qunar/tc/qmq/meta/loadbalance/AbstractLoadBalance.java @@ -0,0 +1,37 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.meta.loadbalance; + +import java.util.List; + +/** + * User: zhaohuiyu Date: 1/9/13 Time: 10:35 AM + */ +public abstract class AbstractLoadBalance implements LoadBalance { + @Override + public List select(String subject, List brokerGroups, int minNum) { + if (brokerGroups == null || brokerGroups.size() == 0) { + return null; + } + if (brokerGroups.size() <= minNum) { + return brokerGroups; + } + return doSelect(subject, brokerGroups, minNum); + } + + abstract List doSelect(String subject, List brokerGroups, int minNum); +} diff --git a/qmq-metaserver/src/main/java/qunar/tc/qmq/meta/loadbalance/LoadBalance.java b/qmq-metaserver/src/main/java/qunar/tc/qmq/meta/loadbalance/LoadBalance.java new file mode 100644 index 00000000..89e9f960 --- /dev/null +++ b/qmq-metaserver/src/main/java/qunar/tc/qmq/meta/loadbalance/LoadBalance.java @@ -0,0 +1,27 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.meta.loadbalance; + +import java.util.List; + +/** + * @author yunfeng.yang + * @since 2017/8/30 + */ +public interface LoadBalance { + List select(String subject, List brokerGroups, int minNum); +} diff --git a/qmq-metaserver/src/main/java/qunar/tc/qmq/meta/loadbalance/RandomLoadBalance.java b/qmq-metaserver/src/main/java/qunar/tc/qmq/meta/loadbalance/RandomLoadBalance.java new file mode 100644 index 00000000..016b7d7b --- /dev/null +++ b/qmq-metaserver/src/main/java/qunar/tc/qmq/meta/loadbalance/RandomLoadBalance.java @@ -0,0 +1,41 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.meta.loadbalance; + +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import java.util.concurrent.ThreadLocalRandom; + +/** + * @author yunfeng.yang + * @since 2017/8/30 + */ +public class RandomLoadBalance extends AbstractLoadBalance { + @Override + List doSelect(String subject, List brokerGroups, int minNum) { + final ThreadLocalRandom random = ThreadLocalRandom.current(); + final Set resultSet = new HashSet<>(minNum); + while (resultSet.size() < minNum) { + final int randomIndex = random.nextInt(brokerGroups.size()); + resultSet.add(brokerGroups.get(randomIndex)); + } + + return new ArrayList<>(resultSet); + } +} diff --git a/qmq-metaserver/src/main/java/qunar/tc/qmq/meta/management/ActionResult.java b/qmq-metaserver/src/main/java/qunar/tc/qmq/meta/management/ActionResult.java new file mode 100644 index 00000000..511156bc --- /dev/null +++ b/qmq-metaserver/src/main/java/qunar/tc/qmq/meta/management/ActionResult.java @@ -0,0 +1,61 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.meta.management; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; + +/** + * @author keli.wang + * @since 2018/7/30 + */ +public class ActionResult { + private final int status; + private final String message; + private final T data; + + @JsonCreator + public ActionResult(@JsonProperty("status") final int status, @JsonProperty("message") final String message, @JsonProperty("data") final T data) { + this.status = status; + this.message = message; + this.data = data; + } + + public static ActionResult error(final String message) { + return new ActionResult<>(-1, message, null); + } + + public static ActionResult error(final String message, final T data) { + return new ActionResult<>(-1, message, data); + } + + public static ActionResult ok(final T data) { + return new ActionResult<>(0, "", data); + } + + public int getStatus() { + return status; + } + + public String getMessage() { + return message; + } + + public T getData() { + return data; + } +} diff --git a/qmq-metaserver/src/main/java/qunar/tc/qmq/meta/management/AddBrokerAction.java b/qmq-metaserver/src/main/java/qunar/tc/qmq/meta/management/AddBrokerAction.java new file mode 100644 index 00000000..3ce81916 --- /dev/null +++ b/qmq-metaserver/src/main/java/qunar/tc/qmq/meta/management/AddBrokerAction.java @@ -0,0 +1,87 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.meta.management; + +import com.google.common.base.Strings; +import qunar.tc.qmq.meta.BrokerRole; +import qunar.tc.qmq.meta.model.BrokerMeta; +import qunar.tc.qmq.meta.store.BrokerStore; + +import javax.servlet.http.HttpServletRequest; +import java.util.Optional; + +/** + * @author keli.wang + * @since 2018-12-03 + */ +public class AddBrokerAction implements MetaManagementAction { + private final BrokerStore store; + + public AddBrokerAction(final BrokerStore store) { + this.store = store; + } + + @Override + public ActionResult handleAction(final HttpServletRequest req) { + try { + final String brokerGroup = req.getParameter("brokerGroup"); + final BrokerRole role = BrokerRole.fromCode(Integer.parseInt(req.getParameter("role"))); + final String hostname = req.getParameter("hostname"); + final String ip = req.getParameter("ip"); + final int servePort = Integer.parseInt(req.getParameter("servePort")); + final int syncPort = Integer.parseInt(req.getParameter("syncPort")); + final BrokerMeta broker = new BrokerMeta(brokerGroup, role, hostname, ip, servePort, syncPort); + + final Optional validateResult = validateBroker(broker); + if (validateResult.isPresent()) { + return ActionResult.error(validateResult.get()); + } + + final int result = store.insertBroker(broker); + if (result > 0) { + return ActionResult.ok(broker); + } else { + return ActionResult.error("broker group's role already exist or broker already added"); + } + } catch (Exception e) { + return ActionResult.error("add broker failed, caused by: " + e.getMessage()); + } + } + + private Optional validateBroker(final BrokerMeta broker) { + if (Strings.isNullOrEmpty(broker.getGroup())) { + return Optional.of("please provide broker group name"); + } + if (broker.getRole() == BrokerRole.STANDBY || broker.getRole() == BrokerRole.DELAY) { + return Optional.of("invalid broker role code " + broker.getRole().getCode()); + } + if (Strings.isNullOrEmpty(broker.getHostname())) { + return Optional.of("please provide broker hostname"); + } + if (Strings.isNullOrEmpty(broker.getIp())) { + return Optional.of("please provide broker ip"); + } + + final int servePort = broker.getServePort(); + final int syncPort = broker.getSyncPort(); + if (servePort <= 0 || syncPort <= 0 || servePort == syncPort) { + return Optional.of("serve port and sync port should valid and should be different port"); + } + + return Optional.empty(); + } +} diff --git a/qmq-metaserver/src/main/java/qunar/tc/qmq/meta/management/AddNewSubjectAction.java b/qmq-metaserver/src/main/java/qunar/tc/qmq/meta/management/AddNewSubjectAction.java new file mode 100644 index 00000000..1e9a6d73 --- /dev/null +++ b/qmq-metaserver/src/main/java/qunar/tc/qmq/meta/management/AddNewSubjectAction.java @@ -0,0 +1,71 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.meta.management; + +import com.google.common.base.Strings; +import com.google.common.collect.ImmutableMap; +import org.springframework.dao.DuplicateKeyException; +import qunar.tc.qmq.meta.model.SubjectInfo; +import qunar.tc.qmq.meta.store.Store; + +import javax.servlet.http.HttpServletRequest; + +/** + * @author keli.wang + * @since 2018/6/19 + */ +public class AddNewSubjectAction implements MetaManagementAction { + private final Store store; + + public AddNewSubjectAction(final Store store) { + this.store = store; + } + + @Override + public Object handleAction(final HttpServletRequest req) { + final String subject = req.getParameter("subject"); + final String tag = req.getParameter("tag"); + + if (Strings.isNullOrEmpty(subject) || Strings.isNullOrEmpty(tag)) { + return ImmutableMap.of( + "error", "必须同时提供 subject 和 tag 两个参数" + ); + } + + final SubjectInfo subjectInfo = store.getSubjectInfo(subject); + if (subjectInfo != null) { + return ImmutableMap.of( + "error", "主题已存在", + "data", subjectInfo + ); + } + + try { + store.insertSubject(subject, tag); + final SubjectInfo info = new SubjectInfo(); + info.setName(subject); + info.setTag(tag); + info.setUpdateTime(System.currentTimeMillis()); + return ImmutableMap.of("data", info); + } catch (DuplicateKeyException e) { + return ImmutableMap.of( + "error", "主题已存在", + "data", store.getSubjectInfo(subject) + ); + } + } +} diff --git a/qmq-metaserver/src/main/java/qunar/tc/qmq/meta/management/AddSubjectBrokerGroupAction.java b/qmq-metaserver/src/main/java/qunar/tc/qmq/meta/management/AddSubjectBrokerGroupAction.java new file mode 100644 index 00000000..8e6756aa --- /dev/null +++ b/qmq-metaserver/src/main/java/qunar/tc/qmq/meta/management/AddSubjectBrokerGroupAction.java @@ -0,0 +1,94 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.meta.management; + +import com.google.common.base.Strings; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Sets; +import org.springframework.dao.EmptyResultDataAccessException; +import qunar.tc.qmq.meta.cache.CachedMetaInfoManager; +import qunar.tc.qmq.meta.model.SubjectRoute; +import qunar.tc.qmq.meta.store.Store; + +import javax.servlet.http.HttpServletRequest; +import java.util.ArrayList; +import java.util.Set; + +/** + * @author keli.wang + * @since 2017/10/20 + */ +public class AddSubjectBrokerGroupAction implements MetaManagementAction { + private final Store store; + private final CachedMetaInfoManager cacheManager; + + public AddSubjectBrokerGroupAction(final Store store, final CachedMetaInfoManager cacheManager) { + this.store = store; + this.cacheManager = cacheManager; + } + + @Override + public Object handleAction(final HttpServletRequest req) { + final String brokerGroup = req.getParameter("brokerGroup"); + final String subject = req.getParameter("subject"); + + if (Strings.isNullOrEmpty(brokerGroup) || Strings.isNullOrEmpty(subject)) { + return ImmutableMap.of( + "error", "必须同时提供 brokerGroup 和 subject 两个参数" + ); + } + + if (!isBrokerGroupExist(brokerGroup)) { + return ImmutableMap.of( + "error", "提供的 brokerGroup 暂时不存在" + ); + } + + final SubjectRoute oldRoute = querySubjectRoute(subject); + if (oldRoute == null) { + return ImmutableMap.of( + "error", "提供的 subject 暂时没有分配过 brokerGroup,无法扩充" + ); + } + + final Set brokerGroups = Sets.newHashSet(oldRoute.getBrokerGroups()); + brokerGroups.add(brokerGroup); + final int affectedRows = store.updateSubjectRoute(subject, oldRoute.getVersion(), new ArrayList<>(brokerGroups)); + cacheManager.executeRefreshTask(); + + if (affectedRows == 1) { + return store.selectSubjectRoute(subject); + } else { + return ImmutableMap.of( + "error", "扩充 subject 对应 brokerGroup 失败,请参考下面的当前配置", + "subjectRoute", store.selectSubjectRoute(subject) + ); + } + } + + private SubjectRoute querySubjectRoute(final String subject) { + try { + return store.selectSubjectRoute(subject); + } catch (EmptyResultDataAccessException ignore) { + return null; + } + } + + private boolean isBrokerGroupExist(final String brokerGroup) { + return store.getBrokerGroup(brokerGroup) != null; + } +} diff --git a/qmq-metaserver/src/main/java/qunar/tc/qmq/meta/management/ExtendSubjectRouteAction.java b/qmq-metaserver/src/main/java/qunar/tc/qmq/meta/management/ExtendSubjectRouteAction.java new file mode 100644 index 00000000..fc1715a8 --- /dev/null +++ b/qmq-metaserver/src/main/java/qunar/tc/qmq/meta/management/ExtendSubjectRouteAction.java @@ -0,0 +1,90 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.meta.management; + +import com.google.common.base.Strings; +import com.google.common.collect.Sets; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import qunar.tc.qmq.meta.cache.CachedMetaInfoManager; +import qunar.tc.qmq.meta.model.SubjectRoute; +import qunar.tc.qmq.meta.store.Store; +import qunar.tc.qmq.meta.utils.SubjectUtils; + +import javax.servlet.http.HttpServletRequest; +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; +import java.util.Set; + +/** + * Created by zhaohui.yu + * 7/24/18 + */ +public class ExtendSubjectRouteAction implements MetaManagementAction { + private static final Logger LOG = LoggerFactory.getLogger(ExtendSubjectRouteAction.class); + + private final Store store; + private final CachedMetaInfoManager cache; + + public ExtendSubjectRouteAction(Store store, CachedMetaInfoManager cache) { + this.store = store; + this.cache = cache; + } + + @Override + public Object handleAction(HttpServletRequest req) { + final String relatedSubject = req.getParameter("relatedSubject"); + final String relatedBrokerGroup = req.getParameter("relatedBrokerGroup"); + final String newBrokerGroup = req.getParameter("newBrokerGroup"); + final boolean isAnySubject = SubjectUtils.isAnySubject(relatedSubject); + + if (Strings.isNullOrEmpty(relatedSubject) + || Strings.isNullOrEmpty(relatedBrokerGroup) + || Strings.isNullOrEmpty(newBrokerGroup)) { + return ActionResult.error("必须同时提供 relatedSubject、relatedBrokerGroup 和 newBrokerGroup 三个参数"); + } + + int updated = 0; + final List routes = store.getAllSubjectRoutes(); + try { + for (SubjectRoute route : routes) { + final String subject = route.getSubject(); + if (!isAnySubject && !Objects.equals(subject, relatedSubject)) { + continue; + } + + final List brokerGroups = route.getBrokerGroups(); + if (!brokerGroups.contains(relatedBrokerGroup)) continue; + if (brokerGroups.contains(newBrokerGroup)) continue; + + final Set newBrokers = Sets.newHashSet(brokerGroups); + newBrokers.add(newBrokerGroup); + + final int affectedRows = store.updateSubjectRoute(subject, route.getVersion(), new ArrayList<>(newBrokers)); + if (affectedRows == 1) { + LOG.info("update subject route: {} from {} to {}", subject, brokerGroups, newBrokers); + updated++; + } + } + } finally { + cache.executeRefreshTask(); + } + + return ActionResult.ok("成功更新" + updated + "个subject的路由"); + } +} diff --git a/qmq-metaserver/src/main/java/qunar/tc/qmq/meta/management/ListBrokerGroupsAction.java b/qmq-metaserver/src/main/java/qunar/tc/qmq/meta/management/ListBrokerGroupsAction.java new file mode 100644 index 00000000..20430b86 --- /dev/null +++ b/qmq-metaserver/src/main/java/qunar/tc/qmq/meta/management/ListBrokerGroupsAction.java @@ -0,0 +1,38 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.meta.management; + +import qunar.tc.qmq.meta.store.Store; + +import javax.servlet.http.HttpServletRequest; + +/** + * @author keli.wang + * @since 2017/10/20 + */ +public class ListBrokerGroupsAction implements MetaManagementAction { + private final Store store; + + public ListBrokerGroupsAction(final Store store) { + this.store = store; + } + + @Override + public Object handleAction(final HttpServletRequest req) { + return store.getAllBrokerGroups(); + } +} diff --git a/qmq-metaserver/src/main/java/qunar/tc/qmq/meta/management/ListBrokersAction.java b/qmq-metaserver/src/main/java/qunar/tc/qmq/meta/management/ListBrokersAction.java new file mode 100644 index 00000000..6a808020 --- /dev/null +++ b/qmq-metaserver/src/main/java/qunar/tc/qmq/meta/management/ListBrokersAction.java @@ -0,0 +1,55 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.meta.management; + +import com.google.common.base.Strings; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import qunar.tc.qmq.meta.model.BrokerMeta; +import qunar.tc.qmq.meta.store.BrokerStore; + +import javax.servlet.http.HttpServletRequest; +import java.util.List; + +/** + * @author keli.wang + * @since 2018-12-03 + */ +public class ListBrokersAction implements MetaManagementAction { + private static final Logger LOG = LoggerFactory.getLogger(ListBrokersAction.class); + + private final BrokerStore store; + + public ListBrokersAction(final BrokerStore store) { + this.store = store; + } + + @Override + public ActionResult> handleAction(final HttpServletRequest req) { + final String brokerGroup = req.getParameter("brokerGroup"); + try { + if (Strings.isNullOrEmpty(brokerGroup)) { + return ActionResult.ok(store.allBrokers()); + } else { + return ActionResult.ok(store.queryBrokers(brokerGroup)); + } + } catch (Exception e) { + LOG.error("list brokers failed.", e); + return ActionResult.error("list brokers action failed"); + } + } +} diff --git a/qmq-metaserver/src/main/java/qunar/tc/qmq/meta/management/ListSubjectRoutesAction.java b/qmq-metaserver/src/main/java/qunar/tc/qmq/meta/management/ListSubjectRoutesAction.java new file mode 100644 index 00000000..963ff246 --- /dev/null +++ b/qmq-metaserver/src/main/java/qunar/tc/qmq/meta/management/ListSubjectRoutesAction.java @@ -0,0 +1,38 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.meta.management; + +import qunar.tc.qmq.meta.store.Store; + +import javax.servlet.http.HttpServletRequest; + +/** + * @author keli.wang + * @since 2017/10/20 + */ +public class ListSubjectRoutesAction implements MetaManagementAction { + private final Store store; + + public ListSubjectRoutesAction(final Store store) { + this.store = store; + } + + @Override + public Object handleAction(HttpServletRequest req) { + return store.getAllSubjectRoutes(); + } +} diff --git a/qmq-metaserver/src/main/java/qunar/tc/qmq/meta/management/MetaManagementAction.java b/qmq-metaserver/src/main/java/qunar/tc/qmq/meta/management/MetaManagementAction.java new file mode 100644 index 00000000..a2fd4e1b --- /dev/null +++ b/qmq-metaserver/src/main/java/qunar/tc/qmq/meta/management/MetaManagementAction.java @@ -0,0 +1,27 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.meta.management; + +import javax.servlet.http.HttpServletRequest; + +/** + * @author keli.wang + * @since 2017/10/20 + */ +public interface MetaManagementAction { + Object handleAction(final HttpServletRequest req); +} diff --git a/qmq-metaserver/src/main/java/qunar/tc/qmq/meta/management/MetaManagementActionSupplier.java b/qmq-metaserver/src/main/java/qunar/tc/qmq/meta/management/MetaManagementActionSupplier.java new file mode 100644 index 00000000..d7c4521d --- /dev/null +++ b/qmq-metaserver/src/main/java/qunar/tc/qmq/meta/management/MetaManagementActionSupplier.java @@ -0,0 +1,44 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.meta.management; + +import java.util.concurrent.ConcurrentHashMap; + +/** + * @author keli.wang + * @since 2017/10/20 + */ +public class MetaManagementActionSupplier { + private static final MetaManagementActionSupplier INSTANCE = new MetaManagementActionSupplier(); + private final ConcurrentHashMap actions; + + private MetaManagementActionSupplier() { + this.actions = new ConcurrentHashMap<>(); + } + + public static MetaManagementActionSupplier getInstance() { + return INSTANCE; + } + + public boolean register(final String name, final MetaManagementAction action) { + return actions.putIfAbsent(name, action) == null; + } + + public MetaManagementAction getAction(final String name) { + return actions.get(name); + } +} diff --git a/qmq-metaserver/src/main/java/qunar/tc/qmq/meta/management/QuerySubjectRouteAction.java b/qmq-metaserver/src/main/java/qunar/tc/qmq/meta/management/QuerySubjectRouteAction.java new file mode 100644 index 00000000..b5c74329 --- /dev/null +++ b/qmq-metaserver/src/main/java/qunar/tc/qmq/meta/management/QuerySubjectRouteAction.java @@ -0,0 +1,59 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.meta.management; + +import com.google.common.base.Strings; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import qunar.tc.qmq.meta.model.SubjectRoute; +import qunar.tc.qmq.meta.store.Store; +import qunar.tc.qmq.meta.utils.SubjectUtils; + +import javax.servlet.http.HttpServletRequest; + +/** + * @author keli.wang + * @since 2018/8/15 + */ +public class QuerySubjectRouteAction implements MetaManagementAction { + private static final Logger LOG = LoggerFactory.getLogger(QuerySubjectRouteAction.class); + + private final Store store; + + public QuerySubjectRouteAction(final Store store) { + this.store = store; + } + + @Override + public Object handleAction(final HttpServletRequest req) { + final String subject = req.getParameter("subject"); + if (Strings.isNullOrEmpty(subject)) { + return ActionResult.error("subject不能为空"); + } + if (SubjectUtils.isAnySubject(subject)) { + return ActionResult.error("subject不能为*通配符"); + } + + try { + final SubjectRoute route = store.selectSubjectRoute(subject); + return ActionResult.ok(route); + } catch (Exception e) { + LOG.error("query subject route failed. subject: {}", subject, e); + return ActionResult.error(e.getMessage()); + } + } +} diff --git a/qmq-metaserver/src/main/java/qunar/tc/qmq/meta/management/RemoveSubjectBrokerGroupAction.java b/qmq-metaserver/src/main/java/qunar/tc/qmq/meta/management/RemoveSubjectBrokerGroupAction.java new file mode 100644 index 00000000..cc681e03 --- /dev/null +++ b/qmq-metaserver/src/main/java/qunar/tc/qmq/meta/management/RemoveSubjectBrokerGroupAction.java @@ -0,0 +1,130 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.meta.management; + +import com.google.common.base.Strings; +import org.springframework.dao.EmptyResultDataAccessException; +import qunar.tc.qmq.meta.cache.CachedMetaInfoManager; +import qunar.tc.qmq.meta.model.SubjectRoute; +import qunar.tc.qmq.meta.store.Store; +import qunar.tc.qmq.meta.utils.SubjectUtils; +import qunar.tc.qmq.meta.web.ResultStatus; + +import javax.servlet.http.HttpServletRequest; +import java.util.*; + +/** + * @author keli.wang + * @since 2017/10/20 + */ +public class RemoveSubjectBrokerGroupAction implements MetaManagementAction { + private final Store store; + private final CachedMetaInfoManager cacheManager; + + public RemoveSubjectBrokerGroupAction(final Store store, final CachedMetaInfoManager cacheManager) { + this.store = store; + this.cacheManager = cacheManager; + } + + @Override + public Object handleAction(final HttpServletRequest req) { + final String brokerGroup = req.getParameter("brokerGroup"); + final String subject = req.getParameter("subject"); + + if (Strings.isNullOrEmpty(brokerGroup) || Strings.isNullOrEmpty(subject)) { + return ActionResult.error("必须同时提供 brokerGroup 和 subject 两个参数"); + } + + if (SubjectUtils.isAnySubject(subject)) { + return removeBrokerGroupForAnySubject(brokerGroup); + } else { + return removeBrokerGroupForOneSubject(subject, brokerGroup); + } + } + + private ActionResult> removeBrokerGroupForAnySubject(final String brokerGroup) { + final Map errors = new HashMap<>(); + + final List routes = store.getAllSubjectRoutes(); + try { + for (final SubjectRoute route : routes) { + final List brokerGroups = route.getBrokerGroups(); + if (!brokerGroups.contains(brokerGroup)) { + continue; + } + + final ActionResult result = removeBrokerGroupFromRoute(brokerGroup, route); + if (result.getStatus() != ResultStatus.OK) { + errors.put(route.getSubject(), result.getMessage()); + } + } + } finally { + cacheManager.executeRefreshTask(); + } + + if (errors.isEmpty()) { + return ActionResult.ok(Collections.emptyMap()); + } else { + return ActionResult.error("部分主题路由缩减失败", errors); + } + } + + private ActionResult> removeBrokerGroupForOneSubject(final String subject, final String brokerGroup) { + final SubjectRoute oldRoute = querySubjectRoute(subject); + if (oldRoute == null) { + return ActionResult.error("提供的 subject 暂时没有分配过 brokerGroup", Collections.emptyMap()); + } + + try { + final ActionResult result = removeBrokerGroupFromRoute(brokerGroup, oldRoute); + if (result.getStatus() == ResultStatus.OK) { + return ActionResult.ok(Collections.emptyMap()); + } else { + return ActionResult.error("路由缩减失败", Collections.singletonMap(subject, result.getMessage())); + } + } finally { + cacheManager.executeRefreshTask(); + } + } + + private SubjectRoute querySubjectRoute(final String subject) { + try { + return store.selectSubjectRoute(subject); + } catch (EmptyResultDataAccessException ignore) { + return null; + } + } + + private ActionResult removeBrokerGroupFromRoute(final String brokerGroup, final SubjectRoute route) { + final List brokerGroups = new ArrayList<>(route.getBrokerGroups()); + if (!brokerGroups.contains(brokerGroup)) { + return ActionResult.ok("ok"); + } + + brokerGroups.remove(brokerGroup); + if (brokerGroups.isEmpty()) { + return ActionResult.error("不允许删除主题的最后一个 brokerGroup"); + } + + final int affectedRows = store.updateSubjectRoute(route.getSubject(), route.getVersion(), brokerGroups); + if (affectedRows == 1) { + return ActionResult.ok("ok"); + } else { + return ActionResult.error("缩减 subject 对应 brokerGroup 失败"); + } + } +} diff --git a/qmq-metaserver/src/main/java/qunar/tc/qmq/meta/management/ReplaceBrokerAction.java b/qmq-metaserver/src/main/java/qunar/tc/qmq/meta/management/ReplaceBrokerAction.java new file mode 100644 index 00000000..203fe039 --- /dev/null +++ b/qmq-metaserver/src/main/java/qunar/tc/qmq/meta/management/ReplaceBrokerAction.java @@ -0,0 +1,99 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.meta.management; + +import com.google.common.base.Strings; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import qunar.tc.qmq.meta.BrokerRole; +import qunar.tc.qmq.meta.model.BrokerMeta; +import qunar.tc.qmq.meta.store.BrokerStore; + +import javax.servlet.http.HttpServletRequest; +import java.util.Optional; + +/** + * @author keli.wang + * @since 2018-12-03 + */ +public class ReplaceBrokerAction implements MetaManagementAction { + private static final Logger LOG = LoggerFactory.getLogger(ReplaceBrokerAction.class); + + private static final int RETRY_REPLACE_COUNT = 20; + + private final BrokerStore store; + + public ReplaceBrokerAction(final BrokerStore store) { + this.store = store; + } + + @Override + public ActionResult handleAction(final HttpServletRequest req) { + try { + final String brokerGroup = req.getParameter("brokerGroup"); + final BrokerRole role = BrokerRole.fromCode(Integer.parseInt(req.getParameter("role"))); + if (Strings.isNullOrEmpty(brokerGroup)) { + return ActionResult.error("should provide broker group name"); + } + + final String hostname = req.getParameter("hostname"); + final String ip = req.getParameter("ip"); + final int servePort = Integer.parseInt(req.getParameter("servePort")); + final int syncPort = Integer.parseInt(req.getParameter("syncPort")); + final BrokerMeta broker = new BrokerMeta(brokerGroup, role, hostname, ip, servePort, syncPort); + + final Optional validateResult = validateBroker(broker); + if (validateResult.isPresent()) { + return ActionResult.error(validateResult.get()); + } + + for (int i = 0; i < RETRY_REPLACE_COUNT; i++) { + final Optional current = store.queryByRole(brokerGroup, role.getCode()); + if (!current.isPresent()) { + return ActionResult.error("no exist broker with this broker group name and role"); + } else { + final int result = store.replaceBrokerByRole(current.get(), broker); + if (result > 0) { + return ActionResult.ok(current.get()); + } + } + } + + return ActionResult.error("replace broker failed. retry count: " + RETRY_REPLACE_COUNT); + } catch (Exception e) { + LOG.error("replace broker failed.", e); + return ActionResult.error("replace broker action failed, caused by: " + e.getMessage()); + } + } + + private Optional validateBroker(final BrokerMeta broker) { + if (Strings.isNullOrEmpty(broker.getHostname())) { + return Optional.of("please provide broker hostname"); + } + if (Strings.isNullOrEmpty(broker.getIp())) { + return Optional.of("please provide broker ip"); + } + + final int servePort = broker.getServePort(); + final int syncPort = broker.getSyncPort(); + if (servePort <= 0 || syncPort <= 0 || servePort == syncPort) { + return Optional.of("serve port and sync port should valid and should be different port"); + } + + return Optional.empty(); + } +} diff --git a/qmq-metaserver/src/main/java/qunar/tc/qmq/meta/management/TokenVerificationAction.java b/qmq-metaserver/src/main/java/qunar/tc/qmq/meta/management/TokenVerificationAction.java new file mode 100644 index 00000000..d887130a --- /dev/null +++ b/qmq-metaserver/src/main/java/qunar/tc/qmq/meta/management/TokenVerificationAction.java @@ -0,0 +1,59 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.meta.management; + +import com.google.common.collect.ImmutableMap; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import qunar.tc.qmq.configuration.DynamicConfig; +import qunar.tc.qmq.configuration.DynamicConfigLoader; + +import javax.servlet.http.HttpServletRequest; + +/** + * @author keli.wang + * @since 2017/10/23 + */ +public class TokenVerificationAction implements MetaManagementAction { + private static final Logger LOG = LoggerFactory.getLogger(TokenVerificationAction.class); + + private static final String TOKEN_HEADER = "X-Api-Token"; + + private static volatile ImmutableMap validApiTokens = ImmutableMap.of(); + + static { + final DynamicConfig config = DynamicConfigLoader.load("valid-api-tokens.properties", false); + config.addListener(conf -> validApiTokens = ImmutableMap.copyOf(conf.asMap())); + } + + private final MetaManagementAction action; + + public TokenVerificationAction(final MetaManagementAction action) { + this.action = action; + } + + @Override + public Object handleAction(final HttpServletRequest req) { + final String token = req.getHeader(TOKEN_HEADER); + if (!validApiTokens.containsKey(token)) { + return ActionResult.error("没有提供合法的 Api Token"); + } + + LOG.info("{} passed token verification.", validApiTokens.get(token)); + return action.handleAction(req); + } +} diff --git a/qmq-metaserver/src/main/java/qunar/tc/qmq/meta/model/BrokerMeta.java b/qmq-metaserver/src/main/java/qunar/tc/qmq/meta/model/BrokerMeta.java new file mode 100644 index 00000000..a0ecb6fe --- /dev/null +++ b/qmq-metaserver/src/main/java/qunar/tc/qmq/meta/model/BrokerMeta.java @@ -0,0 +1,65 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.meta.model; + +import qunar.tc.qmq.meta.BrokerRole; + +/** + * @author keli.wang + * @since 2018-11-29 + */ +public class BrokerMeta { + private final String group; + private final BrokerRole role; + private final String hostname; + private final String ip; + private final int servePort; + private final int syncPort; + + public BrokerMeta(final String group, final BrokerRole role, final String hostname, final String ip, final int servePort, final int syncPort) { + this.group = group; + this.role = role; + this.hostname = hostname; + this.ip = ip; + this.servePort = servePort; + this.syncPort = syncPort; + } + + public String getGroup() { + return group; + } + + public BrokerRole getRole() { + return role; + } + + public String getHostname() { + return hostname; + } + + public String getIp() { + return ip; + } + + public int getServePort() { + return servePort; + } + + public int getSyncPort() { + return syncPort; + } +} diff --git a/qmq-metaserver/src/main/java/qunar/tc/qmq/meta/model/ClientMetaInfo.java b/qmq-metaserver/src/main/java/qunar/tc/qmq/meta/model/ClientMetaInfo.java new file mode 100644 index 00000000..03cc1ebf --- /dev/null +++ b/qmq-metaserver/src/main/java/qunar/tc/qmq/meta/model/ClientMetaInfo.java @@ -0,0 +1,102 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.meta.model; + +/** + * @author yunfeng.yang + * @since 2017/9/25 + */ +public class ClientMetaInfo { + private String subject; + private int clientTypeCode; + private String appCode; + private String room; + private String clientId; + private String consumerGroup; + + public String getSubject() { + return subject; + } + + public void setSubject(String subject) { + this.subject = subject; + } + + public int getClientTypeCode() { + return clientTypeCode; + } + + public void setClientTypeCode(int clientTypeCode) { + this.clientTypeCode = clientTypeCode; + } + + public String getAppCode() { + return appCode; + } + + public void setAppCode(String appCode) { + this.appCode = appCode; + } + + public String getRoom() { + return room; + } + + public void setRoom(String room) { + this.room = room; + } + + public String getClientId() { + return clientId; + } + + public void setClientId(String clientId) { + this.clientId = clientId; + } + + public String getConsumerGroup() { + return consumerGroup; + } + + public void setConsumerGroup(String consumerGroup) { + this.consumerGroup = consumerGroup; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (!(o instanceof ClientMetaInfo)) return false; + + ClientMetaInfo that = (ClientMetaInfo) o; + + if (clientTypeCode != that.clientTypeCode) return false; + if (subject != null ? !subject.equals(that.subject) : that.subject != null) return false; + if (appCode != null ? !appCode.equals(that.appCode) : that.appCode != null) return false; + if (clientId != null ? !clientId.equals(that.clientId) : that.clientId != null) return false; + return consumerGroup != null ? consumerGroup.equals(that.consumerGroup) : that.consumerGroup == null; + } + + @Override + public int hashCode() { + int result = subject != null ? subject.hashCode() : 0; + result = 31 * result + clientTypeCode; + result = 31 * result + (appCode != null ? appCode.hashCode() : 0); + result = 31 * result + (clientId != null ? clientId.hashCode() : 0); + result = 31 * result + (consumerGroup != null ? consumerGroup.hashCode() : 0); + return result; + } +} diff --git a/qmq-metaserver/src/main/java/qunar/tc/qmq/meta/model/ClientOfflineState.java b/qmq-metaserver/src/main/java/qunar/tc/qmq/meta/model/ClientOfflineState.java new file mode 100644 index 00000000..16c2fd72 --- /dev/null +++ b/qmq-metaserver/src/main/java/qunar/tc/qmq/meta/model/ClientOfflineState.java @@ -0,0 +1,71 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.meta.model; + +import qunar.tc.qmq.base.OnOfflineState; + +/** + * yiqun.fan@qunar.com 2018/2/28 + */ +public class ClientOfflineState { + + private long id; + private String clientId; + private String subject; + private String consumerGroup; + private OnOfflineState state; + + public long getId() { + return id; + } + + public void setId(long id) { + this.id = id; + } + + public String getClientId() { + return clientId; + } + + public void setClientId(String clientId) { + this.clientId = clientId; + } + + public String getSubject() { + return subject; + } + + public void setSubject(String subject) { + this.subject = subject; + } + + public String getConsumerGroup() { + return consumerGroup; + } + + public void setConsumerGroup(String consumerGroup) { + this.consumerGroup = consumerGroup; + } + + public OnOfflineState getState() { + return state; + } + + public void setState(OnOfflineState state) { + this.state = state; + } +} diff --git a/qmq-metaserver/src/main/java/qunar/tc/qmq/meta/model/GroupedConsumer.java b/qmq-metaserver/src/main/java/qunar/tc/qmq/meta/model/GroupedConsumer.java new file mode 100644 index 00000000..0d0ea871 --- /dev/null +++ b/qmq-metaserver/src/main/java/qunar/tc/qmq/meta/model/GroupedConsumer.java @@ -0,0 +1,80 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.meta.model; + +import java.util.List; + +/** + * @author keli.wang + * @since 2017/12/5 + */ +public class GroupedConsumer { + private String namespace; + private String prefix; + private String consumerGroup; + private String owner; + private List endPoint; + + public GroupedConsumer(final String namespace, final String prefix, final String consumerGroup) { + this.namespace = namespace; + this.prefix = prefix; + this.consumerGroup = consumerGroup; + } + + public GroupedConsumer() { + } + + public String getNamespace() { + return namespace; + } + + public void setNamespace(String namespace) { + this.namespace = namespace; + } + + public String getPrefix() { + return prefix; + } + + public void setPrefix(String prefix) { + this.prefix = prefix; + } + + public String getConsumerGroup() { + return consumerGroup; + } + + public void setConsumerGroup(String consumerGroup) { + this.consumerGroup = consumerGroup; + } + + public List getEndPoint() { + return endPoint; + } + + public void setEndPoint(List endPoint) { + this.endPoint = endPoint; + } + + public String getOwner() { + return owner; + } + + public void setOwner(String owner) { + this.owner = owner; + } +} diff --git a/qmq-metaserver/src/main/java/qunar/tc/qmq/meta/model/ReadonlyBrokerGroupSetting.java b/qmq-metaserver/src/main/java/qunar/tc/qmq/meta/model/ReadonlyBrokerGroupSetting.java new file mode 100644 index 00000000..33ce5549 --- /dev/null +++ b/qmq-metaserver/src/main/java/qunar/tc/qmq/meta/model/ReadonlyBrokerGroupSetting.java @@ -0,0 +1,47 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.meta.model; + +/** + * @author keli.wang + * @since 2018/7/30 + */ +public class ReadonlyBrokerGroupSetting { + private final String subject; + private final String brokerGroup; + + public ReadonlyBrokerGroupSetting(String subject, String brokerGroup) { + this.subject = subject; + this.brokerGroup = brokerGroup; + } + + public String getSubject() { + return subject; + } + + public String getBrokerGroup() { + return brokerGroup; + } + + @Override + public String toString() { + return "ReadonlyBrokerGroup{" + + "subject='" + subject + '\'' + + ", brokerGroup='" + brokerGroup + '\'' + + '}'; + } +} diff --git a/qmq-metaserver/src/main/java/qunar/tc/qmq/meta/model/SubjectInfo.java b/qmq-metaserver/src/main/java/qunar/tc/qmq/meta/model/SubjectInfo.java new file mode 100644 index 00000000..09f57f4c --- /dev/null +++ b/qmq-metaserver/src/main/java/qunar/tc/qmq/meta/model/SubjectInfo.java @@ -0,0 +1,59 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.meta.model; + +/** + * @author leoliang + */ +public class SubjectInfo { + private String name; + private String tag; + private long updateTime; + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public String getTag() { + return tag; + } + + public void setTag(String tag) { + this.tag = tag; + } + + public long getUpdateTime() { + return updateTime; + } + + public void setUpdateTime(long updateTime) { + this.updateTime = updateTime; + } + + @Override + public String toString() { + return "SubjectInfo{" + + "name='" + name + '\'' + + ", tag='" + tag + '\'' + + ", updateTime=" + updateTime + + '}'; + } +} diff --git a/qmq-metaserver/src/main/java/qunar/tc/qmq/meta/model/SubjectRoute.java b/qmq-metaserver/src/main/java/qunar/tc/qmq/meta/model/SubjectRoute.java new file mode 100644 index 00000000..8a5a5020 --- /dev/null +++ b/qmq-metaserver/src/main/java/qunar/tc/qmq/meta/model/SubjectRoute.java @@ -0,0 +1,62 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.meta.model; + +import java.util.List; + +/** + * @author yunfeng.yang + * @since 2017/8/31 + */ +public class SubjectRoute { + private String subject; + private List brokerGroups; + private int version; + private long updateTime; + + public String getSubject() { + return subject; + } + + public void setSubject(String subject) { + this.subject = subject; + } + + public List getBrokerGroups() { + return brokerGroups; + } + + public void setBrokerGroups(List brokerGroups) { + this.brokerGroups = brokerGroups; + } + + public int getVersion() { + return version; + } + + public void setVersion(int version) { + this.version = version; + } + + public long getUpdateTime() { + return updateTime; + } + + public void setUpdateTime(long updateTime) { + this.updateTime = updateTime; + } +} diff --git a/qmq-metaserver/src/main/java/qunar/tc/qmq/meta/monitor/QMon.java b/qmq-metaserver/src/main/java/qunar/tc/qmq/meta/monitor/QMon.java new file mode 100644 index 00000000..cde80cbb --- /dev/null +++ b/qmq-metaserver/src/main/java/qunar/tc/qmq/meta/monitor/QMon.java @@ -0,0 +1,53 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.meta.monitor; + +import qunar.tc.qmq.metrics.Metrics; + +import static qunar.tc.qmq.metrics.MetricsConstants.SUBJECT_ARRAY; + +/** + * @author yunfeng.yang + * @since 2017/7/10 + */ +public final class QMon { + + public static void brokerRegisterCountInc(String groupName, int requestType) { + Metrics.counter("brokerRegisterCount", new String[]{"groupName", "requestType"}, new String[]{groupName, String.valueOf(requestType)}).inc(); + } + + public static void brokerDisconnectedCountInc(String groupName) { + Metrics.counter("brokerDisconnectedCount", new String[]{"groupName"}, new String[]{groupName}).inc(); + } + + public static void clientRegisterCountInc(String subject, int clientTypeCode) { + Metrics.counter("clientRegisterCount", new String[]{"subject", "clientTypeCode"}, new String[]{subject, String.valueOf(clientTypeCode)}).inc(); + } + + public static void clientSubjectRouteCountInc(String subject) { + Metrics.counter("clientSubjectRouteCount", SUBJECT_ARRAY, new String[]{subject}).inc(); + } + + public static void clientRefreshMetaInfoCountInc(String subject) { + Metrics.counter("clientRefreshMetaInfoCount", SUBJECT_ARRAY, new String[]{subject}).inc(); + } + + public static void subjectInfoNotFound(String subject) { + Metrics.counter("subjectInfoNotFoundCount", SUBJECT_ARRAY, new String[]{subject}).inc(); + } + +} diff --git a/qmq-metaserver/src/main/java/qunar/tc/qmq/meta/processor/BrokerAcquireMetaProcessor.java b/qmq-metaserver/src/main/java/qunar/tc/qmq/meta/processor/BrokerAcquireMetaProcessor.java new file mode 100644 index 00000000..3200ff8b --- /dev/null +++ b/qmq-metaserver/src/main/java/qunar/tc/qmq/meta/processor/BrokerAcquireMetaProcessor.java @@ -0,0 +1,110 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.meta.processor; + +import io.netty.buffer.ByteBuf; +import io.netty.channel.ChannelHandlerContext; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import qunar.tc.qmq.meta.*; +import qunar.tc.qmq.meta.model.BrokerMeta; +import qunar.tc.qmq.meta.store.BrokerStore; +import qunar.tc.qmq.netty.NettyRequestProcessor; +import qunar.tc.qmq.protocol.CommandCode; +import qunar.tc.qmq.protocol.Datagram; +import qunar.tc.qmq.protocol.RemotingCommand; +import qunar.tc.qmq.util.RemotingBuilder; + +import java.util.List; +import java.util.concurrent.CompletableFuture; + +/** + * @author keli.wang + * @since 2017/9/26 + */ +public class BrokerAcquireMetaProcessor implements NettyRequestProcessor { + private static final Logger LOG = LoggerFactory.getLogger(BrokerAcquireMetaProcessor.class); + + private final BrokerStore store; + + public BrokerAcquireMetaProcessor(final BrokerStore store) { + this.store = store; + } + + @Override + public CompletableFuture processRequest(ChannelHandlerContext ctx, RemotingCommand command) { + final ByteBuf body = command.getBody(); + final BrokerAcquireMetaRequest request = BrokerAcquireMetaRequestSerializer.deSerialize(body); + final String hostname = request.getHostname(); + final int port = request.getPort(); + + final String brokerAddress = hostname + "/" + port; + LOG.info("broker request BROKER_ACQUIRE_META: {}", brokerAddress); + + final BrokerAcquireMetaResponse resp = createResponse(hostname, port); + final Datagram datagram = RemotingBuilder.buildResponseDatagram(CommandCode.SUCCESS, command.getHeader(), out -> { + BrokerAcquireMetaResponseSerializer.serialize(resp, out); + }); + + LOG.info("assign {} to {}", resp, brokerAddress); + return CompletableFuture.completedFuture(datagram); + } + + private BrokerAcquireMetaResponse createResponse(final String hostname, final int servePort) { + final BrokerMeta broker = store.queryBroker(hostname, servePort).orElseThrow(() -> new RuntimeException("cannot find broker meta for " + hostname + ":" + servePort)); + final BrokerAcquireMetaResponse resp = new BrokerAcquireMetaResponse(); + resp.setName(broker.getGroup()); + resp.setRole(broker.getRole()); + if (needSync(broker)) { + resp.setMaster(loadSyncAddress(broker)); + } else { + resp.setMaster(""); + } + return resp; + } + + private boolean needSync(final BrokerMeta broker) { + final BrokerRole role = broker.getRole(); + return role == BrokerRole.SLAVE + || role == BrokerRole.DELAY_SLAVE + || role == BrokerRole.BACKUP + || role == BrokerRole.DELAY_BACKUP; + + } + + private String loadSyncAddress(final BrokerMeta broker) { + final List brokers = store.queryBrokers(broker.getGroup()); + for (final BrokerMeta b : brokers) { + if (isMaster(b)) { + return b.getIp() + ":" + b.getSyncPort(); + } + } + + throw new RuntimeException("cannot find master in broker group " + broker.getGroup()); + } + + private boolean isMaster(final BrokerMeta broker) { + final BrokerRole role = broker.getRole(); + return role == BrokerRole.MASTER + || role == BrokerRole.DELAY_MASTER; + } + + @Override + public boolean rejectRequest() { + return false; + } +} diff --git a/qmq-metaserver/src/main/java/qunar/tc/qmq/meta/processor/BrokerRegisterProcessor.java b/qmq-metaserver/src/main/java/qunar/tc/qmq/meta/processor/BrokerRegisterProcessor.java new file mode 100644 index 00000000..254a0d0b --- /dev/null +++ b/qmq-metaserver/src/main/java/qunar/tc/qmq/meta/processor/BrokerRegisterProcessor.java @@ -0,0 +1,168 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.meta.processor; + +import io.netty.buffer.ByteBuf; +import io.netty.channel.ChannelHandlerContext; +import io.netty.util.Timeout; +import io.netty.util.TimerTask; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import qunar.tc.qmq.configuration.DynamicConfig; +import qunar.tc.qmq.meta.*; +import qunar.tc.qmq.meta.cache.CachedMetaInfoManager; +import qunar.tc.qmq.meta.monitor.QMon; +import qunar.tc.qmq.meta.store.Store; +import qunar.tc.qmq.netty.NettyRequestProcessor; +import qunar.tc.qmq.protocol.CommandCode; +import qunar.tc.qmq.protocol.Datagram; +import qunar.tc.qmq.protocol.RemotingCommand; +import qunar.tc.qmq.service.HeartbeatManager; +import qunar.tc.qmq.util.RemotingBuilder; + +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.TimeUnit; + +/** + * @author yunfeng.yang + * @since 2017/8/30 + */ +public class BrokerRegisterProcessor implements NettyRequestProcessor { + private static final Logger LOG = LoggerFactory.getLogger(BrokerRegisterProcessor.class); + + private static final long HEARTBEAT_TIMEOUT_MS = 30 * 1000; + + private final DynamicConfig config; + private final Store store; + private final CachedMetaInfoManager cachedMetaInfoManager; + private final HeartbeatManager heartbeatManager; + + public BrokerRegisterProcessor(DynamicConfig config, CachedMetaInfoManager cachedMetaInfoManager, Store store) { + this.store = store; + this.config = config; + this.cachedMetaInfoManager = cachedMetaInfoManager; + this.heartbeatManager = new HeartbeatManager<>(); + } + + @Override + public CompletableFuture processRequest(ChannelHandlerContext ctx, RemotingCommand request) { + final ByteBuf body = request.getBody(); + final BrokerRegisterRequest brokerRequest = BrokerRegisterRequestSerializer.deSerialize(body); + final String groupName = brokerRequest.getGroupName(); + final int brokerRole = brokerRequest.getBrokerRole(); + final int requestType = brokerRequest.getRequestType(); + + QMon.brokerRegisterCountInc(groupName, requestType); + + LOG.info("broker register request received. request: {}", brokerRequest); + + if (brokerRole == BrokerRole.SLAVE.getCode() || brokerRole == BrokerRole.DELAY_SLAVE.getCode()) { + return CompletableFuture.completedFuture(handleSlave(request)); + } + + return CompletableFuture.completedFuture(handleMaster(request, brokerRequest)); + } + + private Datagram handleSlave(final Datagram request) { + return RemotingBuilder.buildEmptyResponseDatagram(CommandCode.SUCCESS, request.getHeader()); + } + + private Datagram handleMaster(final Datagram request, final BrokerRegisterRequest brokerRequest) { + final int requestType = brokerRequest.getRequestType(); + + if (requestType == BrokerRequestType.HEARTBEAT.getCode()) { + return handleHeartbeat(request, brokerRequest); + } else if (requestType == BrokerRequestType.ONLINE.getCode()) { + return handleOnline(request, brokerRequest); + } else if (requestType == BrokerRequestType.OFFLINE.getCode()) { + return handleOffline(request, brokerRequest); + } + + throw new RuntimeException("unsupported request type " + requestType); + } + + private Datagram handleHeartbeat(final Datagram request, final BrokerRegisterRequest brokerRequest) { + final String groupName = brokerRequest.getGroupName(); + final int brokerState = brokerRequest.getBrokerState(); + + refreshHeartbeat(groupName); + + final BrokerGroup brokerGroupInCache = cachedMetaInfoManager.getBrokerGroup(groupName); + if (brokerGroupInCache == null || brokerState != brokerGroupInCache.getBrokerState().getCode()) { + final BrokerGroup groupInStore = store.getBrokerGroup(groupName); + if (groupInStore != null && groupInStore.getBrokerState().getCode() != brokerState) { + store.updateBrokerGroup(groupName, BrokerState.codeOf(brokerState)); + } + } + + LOG.info("Broker heartbeat response, request:{}", brokerRequest); + return RemotingBuilder.buildEmptyResponseDatagram(CommandCode.SUCCESS, request.getHeader()); + } + + private Datagram handleOnline(final Datagram request, final BrokerRegisterRequest brokerRequest) { + final String groupName = brokerRequest.getGroupName(); + final String brokerAddress = brokerRequest.getBrokerAddress(); + final BrokerGroupKind kind = BrokerRole.fromCode(brokerRequest.getBrokerRole()).getKind(); + + refreshHeartbeat(groupName); + + store.insertOrUpdateBrokerGroup(groupName, kind, brokerAddress, BrokerState.RW); + cachedMetaInfoManager.executeRefreshTask(); + LOG.info("Broker online success, request:{}", brokerRequest); + return RemotingBuilder.buildEmptyResponseDatagram(CommandCode.SUCCESS, request.getHeader()); + } + + private Datagram handleOffline(final Datagram request, final BrokerRegisterRequest brokerRequest) { + final String groupName = brokerRequest.getGroupName(); + final String brokerAddress = brokerRequest.getBrokerAddress(); + final BrokerGroupKind kind = BrokerRole.fromCode(brokerRequest.getBrokerRole()).getKind(); + + store.insertOrUpdateBrokerGroup(groupName, kind, brokerAddress, BrokerState.NRW); + cachedMetaInfoManager.executeRefreshTask(); + LOG.info("broker offline success, request:{}", brokerRequest); + return RemotingBuilder.buildEmptyResponseDatagram(CommandCode.SUCCESS, request.getHeader()); + } + + private void refreshHeartbeat(String groupName) { + heartbeatManager.cancel(groupName); + final HeartbeatTimerTask heartbeatTimerTask = new HeartbeatTimerTask(store, groupName); + final long timeoutMs = config.getLong("heartbeat.timeout.ms", HEARTBEAT_TIMEOUT_MS); + heartbeatManager.refreshHeartbeat(groupName, heartbeatTimerTask, timeoutMs, TimeUnit.MILLISECONDS); + } + + @Override + public boolean rejectRequest() { + return false; + } + + private static class HeartbeatTimerTask implements TimerTask { + private final Store store; + private final String groupName; + + private HeartbeatTimerTask(final Store store, final String groupName) { + this.store = store; + this.groupName = groupName; + } + + @Override + public void run(Timeout timeout) { + QMon.brokerDisconnectedCountInc(groupName); + LOG.warn("broker group lost connection, groupName:{}", groupName); + store.updateBrokerGroup(groupName, BrokerState.NRW); + } + } +} diff --git a/qmq-metaserver/src/main/java/qunar/tc/qmq/meta/processor/ClientRegisterProcessor.java b/qmq-metaserver/src/main/java/qunar/tc/qmq/meta/processor/ClientRegisterProcessor.java new file mode 100644 index 00000000..10592828 --- /dev/null +++ b/qmq-metaserver/src/main/java/qunar/tc/qmq/meta/processor/ClientRegisterProcessor.java @@ -0,0 +1,96 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.meta.processor; + +import io.netty.buffer.ByteBuf; +import io.netty.channel.ChannelHandlerContext; +import qunar.tc.qmq.meta.cache.AliveClientManager; +import qunar.tc.qmq.meta.cache.CachedOfflineStateManager; +import qunar.tc.qmq.meta.monitor.QMon; +import qunar.tc.qmq.meta.route.SubjectRouter; +import qunar.tc.qmq.meta.store.Store; +import qunar.tc.qmq.netty.NettyRequestProcessor; +import qunar.tc.qmq.protocol.Datagram; +import qunar.tc.qmq.protocol.RemotingCommand; +import qunar.tc.qmq.protocol.RemotingHeader; +import qunar.tc.qmq.protocol.consumer.MetaInfoRequest; +import qunar.tc.qmq.utils.PayloadHolderUtils; + +import java.util.concurrent.CompletableFuture; + +/** + * @author yunfeng.yang + * @since 2017/8/30 + */ +public class ClientRegisterProcessor implements NettyRequestProcessor { + + private final ClientRegisterWorker clientRegisterWorker; + private final AliveClientManager aliveClientManager; + + public ClientRegisterProcessor(final SubjectRouter subjectRouter, + final CachedOfflineStateManager offlineStateManager, + final Store store) { + this.clientRegisterWorker = new ClientRegisterWorker(subjectRouter, offlineStateManager, store); + this.aliveClientManager = AliveClientManager.getInstance(); + } + + @Override + public CompletableFuture processRequest(ChannelHandlerContext ctx, RemotingCommand command) { + final RemotingHeader header = command.getHeader(); + final ByteBuf body = command.getBody(); + final MetaInfoRequest request = deserialize(body); + QMon.clientRegisterCountInc(request.getSubject(), request.getClientTypeCode()); + + aliveClientManager.renew(request); + clientRegisterWorker.register(new ClientRegisterMessage(request, ctx, header)); + return null; + } + + @SuppressWarnings("unchecked") + private MetaInfoRequest deserialize(ByteBuf buf) { + return new MetaInfoRequest(PayloadHolderUtils.readStringHashMap(buf)); + } + + @Override + public boolean rejectRequest() { + return false; + } + + static class ClientRegisterMessage { + private final RemotingHeader header; + private final MetaInfoRequest metaInfoRequest; + private final ChannelHandlerContext ctx; + + ClientRegisterMessage(MetaInfoRequest metaInfoRequest, ChannelHandlerContext ctx, RemotingHeader header) { + this.metaInfoRequest = metaInfoRequest; + this.ctx = ctx; + this.header = header; + } + + RemotingHeader getHeader() { + return header; + } + + MetaInfoRequest getMetaInfoRequest() { + return metaInfoRequest; + } + + ChannelHandlerContext getCtx() { + return ctx; + } + } +} diff --git a/qmq-metaserver/src/main/java/qunar/tc/qmq/meta/processor/ClientRegisterWorker.java b/qmq-metaserver/src/main/java/qunar/tc/qmq/meta/processor/ClientRegisterWorker.java new file mode 100644 index 00000000..8d81a235 --- /dev/null +++ b/qmq-metaserver/src/main/java/qunar/tc/qmq/meta/processor/ClientRegisterWorker.java @@ -0,0 +1,166 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.meta.processor; + +import io.netty.buffer.ByteBuf; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import qunar.tc.qmq.base.ClientRequestType; +import qunar.tc.qmq.base.OnOfflineState; +import qunar.tc.qmq.concurrent.ActorSystem; +import qunar.tc.qmq.meta.BrokerCluster; +import qunar.tc.qmq.meta.BrokerGroup; +import qunar.tc.qmq.meta.BrokerState; +import qunar.tc.qmq.meta.cache.CachedOfflineStateManager; +import qunar.tc.qmq.meta.route.SubjectRouter; +import qunar.tc.qmq.meta.store.Store; +import qunar.tc.qmq.meta.utils.ClientLogUtils; +import qunar.tc.qmq.protocol.CommandCode; +import qunar.tc.qmq.protocol.Datagram; +import qunar.tc.qmq.protocol.PayloadHolder; +import qunar.tc.qmq.protocol.RemotingHeader; +import qunar.tc.qmq.protocol.consumer.MetaInfoRequest; +import qunar.tc.qmq.protocol.consumer.MetaInfoResponse; +import qunar.tc.qmq.util.RemotingBuilder; +import qunar.tc.qmq.utils.PayloadHolderUtils; +import qunar.tc.qmq.utils.RetrySubjectUtils; + +import java.util.ArrayList; +import java.util.List; + +/** + * @author yunfeng.yang + * @since 2017/9/1 + */ +class ClientRegisterWorker implements ActorSystem.Processor { + private static final Logger LOG = LoggerFactory.getLogger(ClientRegisterProcessor.class); + + private final SubjectRouter subjectRouter; + private final ActorSystem actorSystem; + private final Store store; + private final CachedOfflineStateManager offlineStateManager; + + ClientRegisterWorker(final SubjectRouter subjectRouter, final CachedOfflineStateManager offlineStateManager, final Store store) { + this.subjectRouter = subjectRouter; + this.actorSystem = new ActorSystem("qmq-meta"); + this.offlineStateManager = offlineStateManager; + this.store = store; + } + + void register(ClientRegisterProcessor.ClientRegisterMessage message) { + actorSystem.dispatch("client-register-" + message.getMetaInfoRequest().getSubject(), message, this); + } + + @Override + public boolean process(ClientRegisterProcessor.ClientRegisterMessage message, ActorSystem.Actor self) { + final MetaInfoRequest request = message.getMetaInfoRequest(); + + final MetaInfoResponse response = handleClientRegister(request); + writeResponse(message, response); + return true; + } + + private MetaInfoResponse handleClientRegister(final MetaInfoRequest request) { + final String realSubject = RetrySubjectUtils.getRealSubject(request.getSubject()); + final int clientRequestType = request.getRequestType(); + + try { + if (ClientRequestType.ONLINE.getCode() == clientRequestType) { + store.insertClientMetaInfo(request); + } + + final List brokerGroups = subjectRouter.route(realSubject, request); + final List filteredBrokerGroups = filterBrokerGroups(brokerGroups); + final OnOfflineState clientState = offlineStateManager.queryClientState(request.getClientId(), request.getSubject(), request.getConsumerGroup()); + + ClientLogUtils.log(realSubject, + "client register response, request:{}, realSubject:{}, brokerGroups:{}, clientState:{}", + request, realSubject, filteredBrokerGroups, clientState); + + return buildResponse(request, offlineStateManager.getLastUpdateTimestamp(), clientState, new BrokerCluster(filteredBrokerGroups)); + } catch (Exception e) { + LOG.error("process exception. {}", request, e); + return buildResponse(request, -2, OnOfflineState.OFFLINE, new BrokerCluster(new ArrayList<>())); + } + } + + private List filterBrokerGroups(final List brokerGroups) { + return removeNrwBrokerGroup(brokerGroups); + } + + private List removeNrwBrokerGroup(final List brokerGroups) { + if (brokerGroups.isEmpty()) { + return brokerGroups; + } + + final List result = new ArrayList<>(); + for (final BrokerGroup brokerGroup : brokerGroups) { + if (brokerGroup.getBrokerState() != BrokerState.NRW) { + result.add(brokerGroup); + } + } + return result; + } + + private MetaInfoResponse buildResponse(MetaInfoRequest clientRequest, long updateTime, OnOfflineState clientState, BrokerCluster brokerCluster) { + final MetaInfoResponse response = new MetaInfoResponse(); + response.setTimestamp(updateTime); + response.setOnOfflineState(clientState); + response.setSubject(clientRequest.getSubject()); + response.setConsumerGroup(clientRequest.getConsumerGroup()); + response.setClientTypeCode(clientRequest.getClientTypeCode()); + response.setBrokerCluster(brokerCluster); + return response; + } + + private void writeResponse(final ClientRegisterProcessor.ClientRegisterMessage message, final MetaInfoResponse response) { + final RemotingHeader header = message.getHeader(); + final MetaInfoResponsePayloadHolder payloadHolder = new MetaInfoResponsePayloadHolder(response, header.getVersion()); + final Datagram datagram = RemotingBuilder.buildResponseDatagram(CommandCode.SUCCESS, header, payloadHolder); + message.getCtx().writeAndFlush(datagram); + } + + private static class MetaInfoResponsePayloadHolder implements PayloadHolder { + private final MetaInfoResponse response; + private final short version; + + MetaInfoResponsePayloadHolder(MetaInfoResponse response, short version) { + this.response = response; + this.version = version; + } + + @Override + public void writeBody(ByteBuf out) { + out.writeLong(response.getTimestamp()); + PayloadHolderUtils.writeString(response.getSubject(), out); + PayloadHolderUtils.writeString(response.getConsumerGroup(), out); + out.writeByte(response.getOnOfflineState().code()); + out.writeByte(response.getClientTypeCode()); + out.writeShort(response.getBrokerCluster().getBrokerGroups().size()); + writeBrokerCluster(out); + } + + private void writeBrokerCluster(ByteBuf out) { + for (BrokerGroup brokerGroup : response.getBrokerCluster().getBrokerGroups()) { + PayloadHolderUtils.writeString(brokerGroup.getGroupName(), out); + PayloadHolderUtils.writeString(brokerGroup.getMaster(), out); + out.writeLong(brokerGroup.getUpdateTime()); + out.writeByte(brokerGroup.getBrokerState().getCode()); + } + } + } +} diff --git a/qmq-metaserver/src/main/java/qunar/tc/qmq/meta/route/SubjectConsumerService.java b/qmq-metaserver/src/main/java/qunar/tc/qmq/meta/route/SubjectConsumerService.java new file mode 100644 index 00000000..28ef2112 --- /dev/null +++ b/qmq-metaserver/src/main/java/qunar/tc/qmq/meta/route/SubjectConsumerService.java @@ -0,0 +1,29 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.meta.route; + +import qunar.tc.qmq.meta.model.GroupedConsumer; + +import java.util.List; + +/** + * @author keli.wang + * @since 2017/12/5 + */ +public interface SubjectConsumerService { + List consumers(final String subject); +} diff --git a/qmq-metaserver/src/main/java/qunar/tc/qmq/meta/route/SubjectRegisterService.java b/qmq-metaserver/src/main/java/qunar/tc/qmq/meta/route/SubjectRegisterService.java new file mode 100644 index 00000000..1e2ad6ba --- /dev/null +++ b/qmq-metaserver/src/main/java/qunar/tc/qmq/meta/route/SubjectRegisterService.java @@ -0,0 +1,25 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.meta.route; + +/** + * @author keli.wang + * @since 2017/12/4 + */ +public interface SubjectRegisterService { + void register(final String subject, final String tenant, final String appCode); +} diff --git a/qmq-metaserver/src/main/java/qunar/tc/qmq/meta/route/SubjectRouter.java b/qmq-metaserver/src/main/java/qunar/tc/qmq/meta/route/SubjectRouter.java new file mode 100644 index 00000000..666b9803 --- /dev/null +++ b/qmq-metaserver/src/main/java/qunar/tc/qmq/meta/route/SubjectRouter.java @@ -0,0 +1,30 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.meta.route; + +import qunar.tc.qmq.meta.BrokerGroup; +import qunar.tc.qmq.protocol.consumer.MetaInfoRequest; + +import java.util.List; + +/** + * @author keli.wang + * @since 2017/12/4 + */ +public interface SubjectRouter { + List route(final String realSubject, final MetaInfoRequest request); +} diff --git a/qmq-metaserver/src/main/java/qunar/tc/qmq/meta/route/impl/DefaultSubjectRouter.java b/qmq-metaserver/src/main/java/qunar/tc/qmq/meta/route/impl/DefaultSubjectRouter.java new file mode 100644 index 00000000..71c685e9 --- /dev/null +++ b/qmq-metaserver/src/main/java/qunar/tc/qmq/meta/route/impl/DefaultSubjectRouter.java @@ -0,0 +1,184 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.meta.route.impl; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import qunar.tc.qmq.common.ClientType; +import qunar.tc.qmq.configuration.DynamicConfig; +import qunar.tc.qmq.meta.BrokerGroup; +import qunar.tc.qmq.meta.cache.CachedMetaInfoManager; +import qunar.tc.qmq.meta.loadbalance.LoadBalance; +import qunar.tc.qmq.meta.loadbalance.RandomLoadBalance; +import qunar.tc.qmq.meta.model.SubjectInfo; +import qunar.tc.qmq.meta.model.SubjectRoute; +import qunar.tc.qmq.meta.monitor.QMon; +import qunar.tc.qmq.meta.route.SubjectRouter; +import qunar.tc.qmq.meta.store.Store; +import qunar.tc.qmq.protocol.consumer.MetaInfoRequest; + +import java.util.*; + +/** + * @author keli.wang + * @since 2017/12/4 + */ +public class DefaultSubjectRouter implements SubjectRouter { + private static final Logger LOG = LoggerFactory.getLogger(DefaultSubjectRouter.class); + + private static final int MIN_SUBJECT_ROUTE_VERSION = 1; + private static final int DEFAULT_MIN_NUM = 2; + private static final int MAX_UPDATE_RETRY_TIMES = 5; + + private final CachedMetaInfoManager cachedMetaInfoManager; + private final Store store; + private final LoadBalance loadBalance; + private int minGroupNum = DEFAULT_MIN_NUM; + + public DefaultSubjectRouter(final DynamicConfig config, final CachedMetaInfoManager cachedMetaInfoManager, final Store store) { + this.cachedMetaInfoManager = cachedMetaInfoManager; + this.store = store; + this.loadBalance = new RandomLoadBalance<>(); + + config.addListener(conf -> minGroupNum = conf.getInt("min.group.num", DEFAULT_MIN_NUM)); + } + + @Override + public List route(final String subject, final MetaInfoRequest request) { + try { + QMon.clientSubjectRouteCountInc(subject); + return doRoute(subject, request.getClientTypeCode()); + } catch (Throwable e) { + LOG.error("find subject route error", e); + return Collections.emptyList(); + } + } + + private List doRoute(String subject, int clientTypeCode) { + SubjectInfo subjectInfo = cachedMetaInfoManager.getSubjectInfo(subject); + if (subjectInfo == null) { + // just add monitor event, will use broker_groups with default tag + QMon.subjectInfoNotFound(subject); + subjectInfo = new SubjectInfo(); + subjectInfo.setName(subject); + } + + //query assigned brokers + final List cachedGroupNames = cachedMetaInfoManager.getGroups(subject); + + List routeBrokerGroupNames; + if (cachedGroupNames == null || cachedGroupNames.size() == 0) { + routeBrokerGroupNames = assignNewBrokers(subjectInfo, clientTypeCode); + } else { + routeBrokerGroupNames = reQueryAssignedBrokers(subjectInfo, cachedGroupNames, clientTypeCode); + } + + return selectExistedBrokerGroups(routeBrokerGroupNames); + } + + private List assignNewBrokers(SubjectInfo subjectInfo, int clientTypeCode) { + if (clientTypeCode == ClientType.CONSUMER.getCode()) { + return Collections.emptyList(); + } + + String subject = subjectInfo.getName(); + final List brokerGroupNames = findBrokerGroupNamesFromCache(subjectInfo.getTag()); + final List loadBalanceSelect = loadBalance.select(subject, brokerGroupNames, minGroupNum); + final int affected = store.insertSubjectRoute(subject, MIN_SUBJECT_ROUTE_VERSION, loadBalanceSelect); + if (affected == 1) { + return loadBalanceSelect; + } + + return findOrUpdateInStore(subjectInfo); + } + + private List reQueryAssignedBrokers(SubjectInfo subjectInfo, List cachedGroupNames, int clientTypeCode) { + if (clientTypeCode == ClientType.CONSUMER.getCode()) { + return cachedGroupNames; + } + + if (cachedGroupNames.size() >= minGroupNum) { + return cachedGroupNames; + } + + return findOrUpdateInStore(subjectInfo); + } + + private List findOrUpdateInStore(final SubjectInfo subjectInfo) { + String subject = subjectInfo.getName(); + + int tries = 0; + + while (tries++ < MAX_UPDATE_RETRY_TIMES) { + final SubjectRoute subjectRoute = findSubjectRouteInStore(subject); + final List oldBrokerGroupNames = subjectRoute.getBrokerGroups(); + if (oldBrokerGroupNames.size() >= minGroupNum) { + return oldBrokerGroupNames; + } + + final List brokerGroupNames = findBrokerGroupNamesFromCache(subjectInfo.getTag()); + if (brokerGroupNames.size() < minGroupNum) { + return oldBrokerGroupNames; + } + + final List select = loadBalance.select(subject, brokerGroupNames, minGroupNum); + final List merge = merge(oldBrokerGroupNames, select); + final int affected = store.updateSubjectRoute(subject, subjectRoute.getVersion(), merge); + if (affected == 1) { + return select; + } + } + throw new RuntimeException("find same room subject route error"); + } + + private List merge(List oldBrokerGroupNames, List select) { + final Set merge = new HashSet<>(); + merge.addAll(oldBrokerGroupNames); + merge.addAll(select); + return new ArrayList<>(merge); + } + + private SubjectRoute findSubjectRouteInStore(String subject) { + return store.selectSubjectRoute(subject); + } + + private List findBrokerGroupNamesFromCache(String tag) { + List brokerGroupNames = cachedMetaInfoManager.getAllBrokerGroupNamesByTag(tag); + if (brokerGroupNames == null || brokerGroupNames.isEmpty()) { + brokerGroupNames = cachedMetaInfoManager.getAllDefaultTagBrokerGroupNames(); + } + + if (brokerGroupNames == null || brokerGroupNames.isEmpty()) { + throw new RuntimeException("no broker groups"); + } + return brokerGroupNames; + } + + private List selectExistedBrokerGroups(final List cachedGroupNames) { + if (cachedGroupNames == null || cachedGroupNames.isEmpty()) { + return Collections.emptyList(); + } + final List cachedBrokerGroups = new ArrayList<>(); + for (String groupName : cachedGroupNames) { + final BrokerGroup brokerGroup = cachedMetaInfoManager.getBrokerGroup(groupName); + if (brokerGroup != null) { + cachedBrokerGroups.add(brokerGroup); + } + } + return cachedBrokerGroups; + } +} diff --git a/qmq-metaserver/src/main/java/qunar/tc/qmq/meta/route/impl/DelayRouter.java b/qmq-metaserver/src/main/java/qunar/tc/qmq/meta/route/impl/DelayRouter.java new file mode 100644 index 00000000..6be4e6f3 --- /dev/null +++ b/qmq-metaserver/src/main/java/qunar/tc/qmq/meta/route/impl/DelayRouter.java @@ -0,0 +1,87 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.meta.route.impl; + +import qunar.tc.qmq.meta.BrokerGroup; +import qunar.tc.qmq.meta.BrokerState; +import qunar.tc.qmq.common.ClientType; +import qunar.tc.qmq.meta.cache.CachedMetaInfoManager; +import qunar.tc.qmq.meta.route.SubjectRouter; +import qunar.tc.qmq.protocol.consumer.MetaInfoRequest; + +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import java.util.concurrent.ThreadLocalRandom; +import java.util.stream.Collectors; + +/** + * @author yiqun.fan create on 17-12-6. + */ +public class DelayRouter implements SubjectRouter { + private static final int DEFAULT_MIN_NUM = 2; + private final CachedMetaInfoManager cachedMetaInfoManager; + private final SubjectRouter internal; + + public DelayRouter(CachedMetaInfoManager cachedMetaInfoManager, SubjectRouter internal) { + this.cachedMetaInfoManager = cachedMetaInfoManager; + this.internal = internal; + } + + @Override + public List route(String realSubject, MetaInfoRequest request) { + if (request.getClientTypeCode() == ClientType.DELAY_PRODUCER.getCode()) { + return doRoute(); + } else { + return internal.route(realSubject, request); + } + } + + private List doRoute() { + final List delayGroups = cachedMetaInfoManager.getDelayNewGroups(); + final List filterDelayGroups = filterNrwBrokers(delayGroups); + return select(filterDelayGroups); + } + + private List filterNrwBrokers(final List groups) { + if (groups.isEmpty()) { + return groups; + } + + return groups.stream().filter(group -> group.getBrokerState() != BrokerState.NRW) + .collect(Collectors.toList()); + } + + private List select(final List groups) { + if (groups == null || groups.size() == 0) { + return null; + } + if (groups.size() <= DEFAULT_MIN_NUM) { + return groups; + } + + final ThreadLocalRandom random = ThreadLocalRandom.current(); + final Set resultSet = new HashSet<>(DEFAULT_MIN_NUM); + while (resultSet.size() <= DEFAULT_MIN_NUM) { + final int randomIndex = random.nextInt(groups.size()); + resultSet.add(groups.get(randomIndex)); + } + + return new ArrayList<>(resultSet); + } +} diff --git a/qmq-metaserver/src/main/java/qunar/tc/qmq/meta/route/impl/SubjectConsumerServiceImpl.java b/qmq-metaserver/src/main/java/qunar/tc/qmq/meta/route/impl/SubjectConsumerServiceImpl.java new file mode 100644 index 00000000..bd0118e7 --- /dev/null +++ b/qmq-metaserver/src/main/java/qunar/tc/qmq/meta/route/impl/SubjectConsumerServiceImpl.java @@ -0,0 +1,68 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.meta.route.impl; + +import qunar.tc.qmq.meta.model.ClientMetaInfo; +import qunar.tc.qmq.meta.model.GroupedConsumer; +import qunar.tc.qmq.meta.route.SubjectConsumerService; +import qunar.tc.qmq.meta.store.ClientMetaInfoStore; +import qunar.tc.qmq.meta.store.impl.ClientMetaInfoStoreImpl; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * @author keli.wang + * @since 2017/12/5 + */ +public class SubjectConsumerServiceImpl implements SubjectConsumerService { + private static final String NEW_QMQ_NAMESPACE = "newqmq"; + + private final ClientMetaInfoStore clientMetaInfoStore = new ClientMetaInfoStoreImpl(); + + @Override + public List consumers(final String subject) { + final List metaInfos = clientMetaInfoStore.queryConsumer(subject); + final Map groupedConsumers = new HashMap<>(); + for (ClientMetaInfo clientMetaInfo : metaInfos) { + GroupedConsumer groupedConsumer = groupedConsumers.get(clientMetaInfo.getConsumerGroup()); + if (groupedConsumer == null) { + groupedConsumer = new GroupedConsumer(NEW_QMQ_NAMESPACE, subject, clientMetaInfo.getConsumerGroup()); + groupedConsumers.put(clientMetaInfo.getConsumerGroup(), groupedConsumer); + } + List endpoints = groupedConsumer.getEndPoint(); + if (endpoints == null) { + endpoints = new ArrayList<>(); + groupedConsumer.setEndPoint(endpoints); + } + endpoints.add(buildNewQmqEndPoint(clientMetaInfo)); + } + return new ArrayList<>(groupedConsumers.values()); + } + + private String buildNewQmqEndPoint(ClientMetaInfo clientMetaInfo) { + return "qmq://" + + clientMetaInfo.getClientId() + + "?" + + "room=" + + clientMetaInfo.getRoom() + + "&appCode=" + + clientMetaInfo.getAppCode(); + } +} diff --git a/qmq-metaserver/src/main/java/qunar/tc/qmq/meta/startup/Bootstrap.java b/qmq-metaserver/src/main/java/qunar/tc/qmq/meta/startup/Bootstrap.java new file mode 100644 index 00000000..85df7a52 --- /dev/null +++ b/qmq-metaserver/src/main/java/qunar/tc/qmq/meta/startup/Bootstrap.java @@ -0,0 +1,47 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.meta.startup; + +import org.eclipse.jetty.server.Server; +import org.eclipse.jetty.servlet.ServletContextHandler; +import qunar.tc.qmq.meta.web.*; + +/** + * @author keli.wang + * @since 2018-12-04 + */ +public class Bootstrap { + public static void main(String[] args) throws Exception { + final ServletContextHandler context = new ServletContextHandler(ServletContextHandler.SESSIONS); + context.setContextPath("/"); + context.setResourceBase(System.getProperty("java.io.tmpdir")); + + final ServerWrapper wrapper = new ServerWrapper(); + wrapper.start(context.getServletContext()); + + context.addServlet(MetaServerAddressSupplierServlet.class, "/meta/address"); + context.addServlet(MetaManagementServlet.class, "/management"); + context.addServlet(SubjectConsumerServlet.class, "/subject/consumers"); + context.addServlet(OnOfflineServlet.class, "/onoffline"); + + // TODO(keli.wang): allow set port use env + final Server server = new Server(8080); + server.setHandler(context); + server.start(); + server.join(); + } +} diff --git a/qmq-metaserver/src/main/java/qunar/tc/qmq/meta/startup/ServerWrapper.java b/qmq-metaserver/src/main/java/qunar/tc/qmq/meta/startup/ServerWrapper.java new file mode 100644 index 00000000..b71b78a6 --- /dev/null +++ b/qmq-metaserver/src/main/java/qunar/tc/qmq/meta/startup/ServerWrapper.java @@ -0,0 +1,116 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.meta.startup; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import qunar.tc.qmq.common.Disposable; +import qunar.tc.qmq.configuration.DynamicConfig; +import qunar.tc.qmq.configuration.DynamicConfigLoader; +import qunar.tc.qmq.meta.cache.CachedMetaInfoManager; +import qunar.tc.qmq.meta.cache.CachedOfflineStateManager; +import qunar.tc.qmq.meta.management.*; +import qunar.tc.qmq.meta.processor.BrokerAcquireMetaProcessor; +import qunar.tc.qmq.meta.processor.BrokerRegisterProcessor; +import qunar.tc.qmq.meta.processor.ClientRegisterProcessor; +import qunar.tc.qmq.meta.route.SubjectRouter; +import qunar.tc.qmq.meta.route.impl.DefaultSubjectRouter; +import qunar.tc.qmq.meta.route.impl.DelayRouter; +import qunar.tc.qmq.meta.store.BrokerStore; +import qunar.tc.qmq.meta.store.JdbcTemplateHolder; +import qunar.tc.qmq.meta.store.Store; +import qunar.tc.qmq.meta.store.impl.BrokerStoreImpl; +import qunar.tc.qmq.meta.store.impl.DatabaseStore; +import qunar.tc.qmq.netty.DefaultConnectionEventHandler; +import qunar.tc.qmq.netty.NettyServer; +import qunar.tc.qmq.protocol.CommandCode; + +import javax.servlet.ServletContext; +import java.util.ArrayList; +import java.util.List; + +/** + * @author yunfeng.yang + * @since 2017/8/30 + */ +@SuppressWarnings("all") +public class ServerWrapper implements Disposable { + private static final Logger LOG = LoggerFactory.getLogger(ServerWrapper.class); + + private static final int DEFAULT_META_SERVER_PORT = 20880; + + private final List resources; + private final DynamicConfig config; + + public ServerWrapper() { + this.resources = new ArrayList<>(); + this.config = DynamicConfigLoader.load("metaserver.properties"); + } + + public void start(ServletContext context) { + final int port = config.getInt("meta.server.port", DEFAULT_META_SERVER_PORT); + context.setAttribute("port", port); + + final Store store = new DatabaseStore(); + final BrokerStore brokerStore = new BrokerStoreImpl(JdbcTemplateHolder.getOrCreate()); + + final CachedMetaInfoManager cachedMetaInfoManager = new CachedMetaInfoManager(config, store); + + final SubjectRouter subjectRouter = createSubjectRouter(cachedMetaInfoManager, store); + final ClientRegisterProcessor clientRegisterProcessor = new ClientRegisterProcessor(subjectRouter, CachedOfflineStateManager.SUPPLIER.get(), store); + final BrokerRegisterProcessor brokerRegisterProcessor = new BrokerRegisterProcessor(config, cachedMetaInfoManager, store); + final BrokerAcquireMetaProcessor brokerAcquireMetaProcessor = new BrokerAcquireMetaProcessor(new BrokerStoreImpl(JdbcTemplateHolder.getOrCreate())); + + final NettyServer metaNettyServer = new NettyServer("meta", Runtime.getRuntime().availableProcessors(), port, new DefaultConnectionEventHandler("meta")); + metaNettyServer.registerProcessor(CommandCode.CLIENT_REGISTER, clientRegisterProcessor); + metaNettyServer.registerProcessor(CommandCode.BROKER_REGISTER, brokerRegisterProcessor); + metaNettyServer.registerProcessor(CommandCode.BROKER_ACQUIRE_META, brokerAcquireMetaProcessor); + metaNettyServer.start(); + + final MetaManagementActionSupplier actions = MetaManagementActionSupplier.getInstance(); + actions.register("AddBroker", new TokenVerificationAction(new AddBrokerAction(brokerStore))); + actions.register("ReplaceBroker", new TokenVerificationAction(new ReplaceBrokerAction(brokerStore))); + actions.register("ListBrokers", new ListBrokersAction(brokerStore)); + actions.register("ListBrokerGroups", new ListBrokerGroupsAction(store)); + actions.register("ListSubjectRoutes", new ListSubjectRoutesAction(store)); + actions.register("AddSubjectBrokerGroup", new TokenVerificationAction(new AddSubjectBrokerGroupAction(store, cachedMetaInfoManager))); + actions.register("RemoveSubjectBrokerGroup", new TokenVerificationAction(new RemoveSubjectBrokerGroupAction(store, cachedMetaInfoManager))); + actions.register("AddNewSubject", new TokenVerificationAction(new AddNewSubjectAction(store))); + actions.register("ExtendSubjectRoute", new TokenVerificationAction(new ExtendSubjectRouteAction(store, cachedMetaInfoManager))); + + resources.add(cachedMetaInfoManager); + resources.add(metaNettyServer); + } + + private SubjectRouter createSubjectRouter(CachedMetaInfoManager cachedMetaInfoManager, Store store) { + return new DelayRouter(cachedMetaInfoManager, new DefaultSubjectRouter(config, cachedMetaInfoManager, store)); + } + + + @Override + public void destroy() { + if (resources.isEmpty()) return; + + for (int i = resources.size() - 1; i >= 0; --i) { + try { + resources.get(i).destroy(); + } catch (Throwable e) { + LOG.error("destroy resource failed", e); + } + } + } +} diff --git a/qmq-metaserver/src/main/java/qunar/tc/qmq/meta/store/BrokerStore.java b/qmq-metaserver/src/main/java/qunar/tc/qmq/meta/store/BrokerStore.java new file mode 100644 index 00000000..7cf3b0f7 --- /dev/null +++ b/qmq-metaserver/src/main/java/qunar/tc/qmq/meta/store/BrokerStore.java @@ -0,0 +1,40 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.meta.store; + +import qunar.tc.qmq.meta.model.BrokerMeta; + +import java.util.List; +import java.util.Optional; + +/** + * @author keli.wang + * @since 2018-11-29 + */ +public interface BrokerStore { + Optional queryBroker(final String hostname, final int servePort); + + Optional queryByRole(final String brokerGroup, final int role); + + List queryBrokers(final String groupName); + + List allBrokers(); + + int insertBroker(final BrokerMeta broker); + + int replaceBrokerByRole(final BrokerMeta oldBroker, final BrokerMeta newBroker); +} diff --git a/qmq-metaserver/src/main/java/qunar/tc/qmq/meta/store/ClientMetaInfoStore.java b/qmq-metaserver/src/main/java/qunar/tc/qmq/meta/store/ClientMetaInfoStore.java new file mode 100644 index 00000000..900caf23 --- /dev/null +++ b/qmq-metaserver/src/main/java/qunar/tc/qmq/meta/store/ClientMetaInfoStore.java @@ -0,0 +1,29 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.meta.store; + +import qunar.tc.qmq.meta.model.ClientMetaInfo; + +import java.util.List; + +/** + * @author keli.wang + * @since 2017/12/5 + */ +public interface ClientMetaInfoStore { + List queryConsumer(final String subject); +} diff --git a/qmq-metaserver/src/main/java/qunar/tc/qmq/meta/store/ClientOfflineStore.java b/qmq-metaserver/src/main/java/qunar/tc/qmq/meta/store/ClientOfflineStore.java new file mode 100644 index 00000000..19363dec --- /dev/null +++ b/qmq-metaserver/src/main/java/qunar/tc/qmq/meta/store/ClientOfflineStore.java @@ -0,0 +1,40 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.meta.store; + +import qunar.tc.qmq.meta.model.ClientOfflineState; + +import java.util.List; +import java.util.Optional; + +/** + * yiqun.fan@qunar.com 2018/2/28 + */ +public interface ClientOfflineStore { + + long now(); + + Optional select(String clientId, String subject, String consumerGroup); + + List selectAll(); + + void insertOrUpdate(ClientOfflineState clientState); + + void delete(String subject, String consumerGroup); + + void delete(String clientId, String subject, String consumerGroup); +} diff --git a/qmq-metaserver/src/main/java/qunar/tc/qmq/meta/store/JdbcTemplateHolder.java b/qmq-metaserver/src/main/java/qunar/tc/qmq/meta/store/JdbcTemplateHolder.java new file mode 100644 index 00000000..75632589 --- /dev/null +++ b/qmq-metaserver/src/main/java/qunar/tc/qmq/meta/store/JdbcTemplateHolder.java @@ -0,0 +1,68 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.meta.store; + +import com.google.common.base.Supplier; +import com.google.common.base.Suppliers; +import com.zaxxer.hikari.HikariConfig; +import com.zaxxer.hikari.HikariDataSource; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.jdbc.datasource.DataSourceTransactionManager; +import org.springframework.transaction.support.TransactionTemplate; +import qunar.tc.qmq.configuration.DynamicConfig; +import qunar.tc.qmq.configuration.DynamicConfigLoader; + +import javax.sql.DataSource; + +/** + * @author keli.wang + * @since 2017/9/28 + */ +public class JdbcTemplateHolder { + private static final Supplier DS_SUPPLIER = Suppliers.memoize(JdbcTemplateHolder::createDataSource); + private static final Supplier SUPPLIER = Suppliers.memoize(JdbcTemplateHolder::createJdbcTemplate); + private static final Supplier TRANS_SUPPLIER = Suppliers.memoize(JdbcTemplateHolder::createTransactionTemplate); + + private static JdbcTemplate createJdbcTemplate() { + return new JdbcTemplate(DS_SUPPLIER.get()); + } + + private static TransactionTemplate createTransactionTemplate() { + return new TransactionTemplate(new DataSourceTransactionManager(DS_SUPPLIER.get())); + } + + private static DataSource createDataSource() { + final DynamicConfig config = DynamicConfigLoader.load("datasource.properties"); + + final HikariConfig cpConfig = new HikariConfig(); + cpConfig.setDriverClassName(config.getString("jdbc.driverClassName", "com.mysql.jdbc.Driver")); + cpConfig.setJdbcUrl(config.getString("jdbc.url")); + cpConfig.setUsername(config.getString("jdbc.username")); + cpConfig.setPassword(config.getString("jdbc.password")); + cpConfig.setMaximumPoolSize(config.getInt("pool.size.max", 10)); + + return new HikariDataSource(cpConfig); + } + + public static JdbcTemplate getOrCreate() { + return SUPPLIER.get(); + } + + public static TransactionTemplate getTransactionOrCreate() { + return TRANS_SUPPLIER.get(); + } +} diff --git a/qmq-metaserver/src/main/java/qunar/tc/qmq/meta/store/Store.java b/qmq-metaserver/src/main/java/qunar/tc/qmq/meta/store/Store.java new file mode 100644 index 00000000..72d33dcd --- /dev/null +++ b/qmq-metaserver/src/main/java/qunar/tc/qmq/meta/store/Store.java @@ -0,0 +1,59 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.meta.store; + +import qunar.tc.qmq.meta.BrokerGroup; +import qunar.tc.qmq.meta.BrokerGroupKind; +import qunar.tc.qmq.meta.BrokerState; +import qunar.tc.qmq.meta.model.SubjectInfo; +import qunar.tc.qmq.meta.model.SubjectRoute; +import qunar.tc.qmq.protocol.consumer.MetaInfoRequest; + +import java.util.List; + +/** + * @author yunfeng.yang + * @since 2017/8/30 + */ +public interface Store { + + int insertSubjectRoute(String subject, int version, List groupNames); + + int updateSubjectRoute(String subject, int version, List groupNames); + + SubjectRoute selectSubjectRoute(String subject); + + void insertOrUpdateBrokerGroup(String groupName, BrokerGroupKind kind, String masterAddress, BrokerState brokerState); + + void updateBrokerGroup(String groupName, BrokerState brokerState); + + void updateBrokerGroupTag(String groupName, String tag); + + void insertSubject(String subject, String tag); + + List getAllBrokerGroups(); + + List getAllSubjectRoutes(); + + List getAllSubjectInfo(); + + SubjectInfo getSubjectInfo(final String subject); + + BrokerGroup getBrokerGroup(String groupName); + + void insertClientMetaInfo(MetaInfoRequest metaInfoRequest); +} diff --git a/qmq-metaserver/src/main/java/qunar/tc/qmq/meta/store/impl/BrokerStoreImpl.java b/qmq-metaserver/src/main/java/qunar/tc/qmq/meta/store/impl/BrokerStoreImpl.java new file mode 100644 index 00000000..b2a086a1 --- /dev/null +++ b/qmq-metaserver/src/main/java/qunar/tc/qmq/meta/store/impl/BrokerStoreImpl.java @@ -0,0 +1,99 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.meta.store.impl; + +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.jdbc.core.RowMapper; +import qunar.tc.qmq.meta.BrokerRole; +import qunar.tc.qmq.meta.model.BrokerMeta; +import qunar.tc.qmq.meta.store.BrokerStore; + +import java.util.List; +import java.util.Optional; + +/** + * @author keli.wang + * @since 2018-11-29 + */ +public class BrokerStoreImpl implements BrokerStore { + private static final String QUERY_BROKER_SQL = "SELECT group_name,role,hostname,INET_NTOA(ip) AS ip,serve_port,sync_port FROM broker WHERE hostname=? AND serve_port=?"; + private static final String QUERY_BY_ROLE_SQL = "SELECT group_name,role,hostname,INET_NTOA(ip) AS ip,serve_port,sync_port FROM broker WHERE group_name=? AND role=?"; + private static final String QUERY_BROKERS_SQL = "SELECT group_name,role,hostname,INET_NTOA(ip) AS ip,serve_port,sync_port FROM broker WHERE group_name=?"; + private static final String ALL_BROKERS_SQL = "SELECT group_name,role,hostname,INET_NTOA(ip) AS ip,serve_port,sync_port FROM broker"; + private static final String INSERT_BROKER_SQL = "INSERT IGNORE INTO broker(group_name,role,hostname,ip,serve_port,sync_port) VALUES (?,?,?,INET_ATON(?),?,?)"; + private static final String REPLACE_BROKER_BY_ROLE_SQL = "UPDATE broker SET hostname=?,ip=INET_ATON(?),serve_port=?,sync_port=? WHERE group_name=? AND role=? AND hostname=? AND serve_port=INET_ATON(?)"; + + private static final RowMapper MAPPER = (rs, i) -> { + final String group = rs.getString("group_name"); + final BrokerRole role = BrokerRole.fromCode(rs.getInt("role")); + final String hostname = rs.getString("hostname"); + final String ip = rs.getString("ip"); + final int servePort = rs.getInt("serve_port"); + final int syncPort = rs.getInt("sync_port"); + return new BrokerMeta(group, role, hostname, ip, servePort, syncPort); + }; + + private final JdbcTemplate jdbcTemplate; + + public BrokerStoreImpl(final JdbcTemplate jdbcTemplate) { + this.jdbcTemplate = jdbcTemplate; + } + + @Override + public Optional queryBroker(final String hostname, final int servePort) { + final List brokers = jdbcTemplate.query(QUERY_BROKER_SQL, MAPPER, hostname, servePort); + return optionalBroker(brokers); + } + + @Override + public Optional queryByRole(final String brokerGroup, final int role) { + final List brokers = jdbcTemplate.query(QUERY_BY_ROLE_SQL, MAPPER, brokerGroup, role); + return optionalBroker(brokers); + } + + private Optional optionalBroker(final List brokers) { + if (brokers.isEmpty()) { + return Optional.empty(); + } else { + return Optional.of(brokers.get(0)); + } + } + + @Override + public List queryBrokers(final String groupName) { + return jdbcTemplate.query(QUERY_BROKERS_SQL, MAPPER, groupName); + } + + @Override + public List allBrokers() { + return jdbcTemplate.query(ALL_BROKERS_SQL, MAPPER); + } + + @Override + public int insertBroker(final BrokerMeta broker) { + return jdbcTemplate.update(INSERT_BROKER_SQL, + broker.getGroup(), broker.getRole().getCode(), + broker.getHostname(), broker.getIp(), broker.getServePort(), broker.getSyncPort()); + } + + @Override + public int replaceBrokerByRole(final BrokerMeta oldBroker, final BrokerMeta newBroker) { + return jdbcTemplate.update(REPLACE_BROKER_BY_ROLE_SQL, + newBroker.getHostname(), newBroker.getIp(), newBroker.getServePort(), newBroker.getSyncPort(), + oldBroker.getGroup(), oldBroker.getRole(), oldBroker.getHostname(), oldBroker.getSyncPort()); + } +} diff --git a/qmq-metaserver/src/main/java/qunar/tc/qmq/meta/store/impl/ClientMetaInfoStoreImpl.java b/qmq-metaserver/src/main/java/qunar/tc/qmq/meta/store/impl/ClientMetaInfoStoreImpl.java new file mode 100644 index 00000000..e9cfcbec --- /dev/null +++ b/qmq-metaserver/src/main/java/qunar/tc/qmq/meta/store/impl/ClientMetaInfoStoreImpl.java @@ -0,0 +1,47 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.meta.store.impl; + +import org.springframework.jdbc.core.JdbcTemplate; +import qunar.tc.qmq.common.ClientType; +import qunar.tc.qmq.meta.store.JdbcTemplateHolder; +import qunar.tc.qmq.meta.model.ClientMetaInfo; +import qunar.tc.qmq.meta.store.ClientMetaInfoStore; + +import java.util.List; + +/** + * @author keli.wang + * @since 2017/12/5 + */ +public class ClientMetaInfoStoreImpl implements ClientMetaInfoStore { + private final JdbcTemplate jdbcTemplate = JdbcTemplateHolder.getOrCreate(); + + @Override + public List queryConsumer(String subject) { + return jdbcTemplate.query("SELECT subject_info,client_type,consumer_group,client_id,app_code,room FROM client_meta_info WHERE subject_info=? AND client_type=?", (rs, rowNum) -> { + final ClientMetaInfo meta = new ClientMetaInfo(); + meta.setSubject(rs.getString("subject_info")); + meta.setClientTypeCode(rs.getInt("client_type")); + meta.setConsumerGroup(rs.getString("consumer_group")); + meta.setClientId(rs.getString("client_id")); + meta.setAppCode(rs.getString("app_code")); + meta.setRoom(rs.getString("room")); + return meta; + }, subject, ClientType.CONSUMER.getCode()); + } +} diff --git a/qmq-metaserver/src/main/java/qunar/tc/qmq/meta/store/impl/ClientOfflineStoreImpl.java b/qmq-metaserver/src/main/java/qunar/tc/qmq/meta/store/impl/ClientOfflineStoreImpl.java new file mode 100644 index 00000000..79907adf --- /dev/null +++ b/qmq-metaserver/src/main/java/qunar/tc/qmq/meta/store/impl/ClientOfflineStoreImpl.java @@ -0,0 +1,110 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.meta.store.impl; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.jdbc.core.RowMapper; +import qunar.tc.qmq.meta.model.ClientOfflineState; +import qunar.tc.qmq.base.OnOfflineState; +import qunar.tc.qmq.meta.store.JdbcTemplateHolder; +import qunar.tc.qmq.meta.store.ClientOfflineStore; + +import java.sql.PreparedStatement; +import java.sql.Statement; +import java.util.List; +import java.util.Optional; + +/** + * yiqun.fan@qunar.com 2018/2/28 + */ +public class ClientOfflineStoreImpl implements ClientOfflineStore { + private static final Logger LOGGER = LoggerFactory.getLogger(ClientOfflineStoreImpl.class); + + private static final String SELECT_NOW_SQL = "SELECT now() as ts"; + private static final String SELECT_SQL = "SELECT id,client_id,subject_info,consumer_group,state FROM client_offline_state WHERE client_id=? and subject_info=? and consumer_group=?"; + private static final String SELECT_ALL_SQL = "SELECT id,client_id,subject_info,consumer_group,state FROM client_offline_state"; + private static final String INSERT_SQL = "INSERT INTO client_offline_state(client_id,subject_info,consumer_group,state) VALUES(?,?,?,?) ON DUPLICATE KEY UPDATE state=?"; + private static final String DELETE_BY_SUBJECT_GROUP_SQL = "DELETE FROM client_offline_state WHERE subject_info=? and consumer_group=?"; + private static final String DELETE_SQL = "DELETE FROM client_offline_state WHERE client_id=? and subject_info=? and consumer_group=?"; + + private static final RowMapper ROW_MAPPER = (rs, rowNum) -> { + try { + ClientOfflineState state = new ClientOfflineState(); + state.setId(rs.getLong("id")); + state.setClientId(rs.getString("client_id")); + state.setSubject(rs.getString("subject_info")); + state.setConsumerGroup(rs.getString("consumer_group")); + state.setState(OnOfflineState.fromCode(rs.getInt("state"))); + return state; + } catch (Exception e) { + LOGGER.error("selectAll exception", e); + return null; + } + }; + + private final JdbcTemplate jdbcTemplate = JdbcTemplateHolder.getOrCreate(); + + @Override + public long now() { + JdbcTemplate jdbcTemplate = JdbcTemplateHolder.getOrCreate(); + List results = jdbcTemplate.query(SELECT_NOW_SQL, (rs, rowNum) -> { + try { + return rs.getTimestamp("ts").getTime(); + } catch (Exception e) { + LOGGER.error("select now exception", e); + return (long) -1; + } + }); + return results != null && !results.isEmpty() ? results.get(0) : -1; + } + + @Override + public Optional select(String clientId, String subject, String consumerGroup) { + List list = jdbcTemplate.query(SELECT_SQL, ROW_MAPPER, clientId, subject, consumerGroup); + return list != null && !list.isEmpty() ? Optional.ofNullable(list.get(0)) : Optional.empty(); + } + + @Override + public List selectAll() { + return jdbcTemplate.query(SELECT_ALL_SQL, ROW_MAPPER); + } + + @Override + public void insertOrUpdate(ClientOfflineState clientState) { + jdbcTemplate.update(con -> { + PreparedStatement ps = con.prepareStatement(INSERT_SQL, Statement.RETURN_GENERATED_KEYS); + ps.setString(1, clientState.getClientId()); + ps.setString(2, clientState.getSubject()); + ps.setString(3, clientState.getConsumerGroup()); + ps.setInt(4, clientState.getState().code()); + ps.setInt(5, clientState.getState().code()); + return ps; + }); + } + + @Override + public void delete(String subject, String consumerGroup) { + jdbcTemplate.update(DELETE_BY_SUBJECT_GROUP_SQL, subject, consumerGroup); + } + + @Override + public void delete(String clientId, String subject, String consumerGroup) { + jdbcTemplate.update(DELETE_SQL, clientId, subject, consumerGroup); + } +} diff --git a/qmq-metaserver/src/main/java/qunar/tc/qmq/meta/store/impl/DatabaseStore.java b/qmq-metaserver/src/main/java/qunar/tc/qmq/meta/store/impl/DatabaseStore.java new file mode 100644 index 00000000..e438e698 --- /dev/null +++ b/qmq-metaserver/src/main/java/qunar/tc/qmq/meta/store/impl/DatabaseStore.java @@ -0,0 +1,194 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.meta.store.impl; + +import com.fasterxml.jackson.core.type.TypeReference; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.dao.EmptyResultDataAccessException; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.jdbc.core.RowMapper; +import qunar.tc.qmq.meta.BrokerGroup; +import qunar.tc.qmq.meta.BrokerGroupKind; +import qunar.tc.qmq.meta.BrokerState; +import qunar.tc.qmq.meta.model.SubjectInfo; +import qunar.tc.qmq.meta.model.SubjectRoute; +import qunar.tc.qmq.meta.store.JdbcTemplateHolder; +import qunar.tc.qmq.meta.store.Store; +import qunar.tc.qmq.protocol.consumer.MetaInfoRequest; + +import java.sql.Timestamp; +import java.util.Collections; +import java.util.Date; +import java.util.List; + +import static qunar.tc.qmq.meta.store.impl.Serializer.serialize; + +/** + * @author yunfeng.yang + * @since 2017/8/31 + */ +public class DatabaseStore implements Store { + private static final Logger LOG = LoggerFactory.getLogger(DatabaseStore.class); + + private static final String INSERT_SUBJECT_ROUTE_SQL = "INSERT IGNORE INTO subject_route(subject_info, version, broker_group_json, create_time) VALUES(?, ?, ?, ?)"; + private static final String UPDATE_SUBJECT_ROUTE_SQL = "UPDATE subject_route SET broker_group_json = ?, version = version + 1 WHERE subject_info = ? AND version = ?"; + private static final String FIND_SUBJECT_ROUTE_SQL = "SELECT subject_info, version, broker_group_json, update_time FROM subject_route"; + private static final String SELECT_SUBJECT_ROUTE_SQL = "SELECT subject_info, version, broker_group_json, update_time FROM subject_route WHERE subject_info = ?"; + + private static final String FIND_BROKER_GROUP_SQL = "SELECT group_name, kind, master_address, broker_state, tag, update_time FROM broker_group"; + private static final String SELECT_BROKER_GROUP_SQL = "SELECT group_name, kind, master_address, broker_state, tag, update_time FROM broker_group WHERE group_name = ?"; + private static final String UPDATE_BROKER_GROUP_SQL = "UPDATE broker_group SET broker_state = ? WHERE group_name = ?"; + private static final String UPDATE_BROKER_GROUP_TAG_SQL = "UPDATE broker_group SET tag = ? WHERE group_name = ?"; + private static final String INSERT_OR_UPDATE_BROKER_GROUP_SQL = "INSERT INTO broker_group(group_name,kind,master_address,broker_state,create_time) VALUES(?,?,?,?,?) ON DUPLICATE KEY UPDATE master_address=?,broker_state=?"; + + private static final String INSERT_CLIENT_META_INFO_SQL = "INSERT IGNORE INTO client_meta_info(subject_info,client_type,consumer_group,client_id,app_code,create_time) VALUES(?, ?, ?, ?, ?, ?)"; + + private static final String INSERT_SUBJECT_INFO_SQL = "INSERT INTO subject_info(name,tag,create_time) VALUES(?,?,?)"; + private static final String ALL_SUBJECT_INFO_SQL = "SELECT name, tag, update_time FROM subject_info"; + private static final String QUERY_SUBJECT_INFO_SQL = "SELECT name, tag, update_time FROM subject_info WHERE name=?"; + + private static final RowMapper BROKER_GROUP_ROW_MAPPER = (rs, rowNum) -> { + final BrokerGroup brokerGroup = new BrokerGroup(); + brokerGroup.setGroupName(rs.getString("group_name")); + brokerGroup.setMaster(rs.getString("master_address")); + brokerGroup.setBrokerState(BrokerState.codeOf(rs.getInt("broker_state"))); + brokerGroup.setUpdateTime(rs.getTimestamp("update_time").getTime()); + brokerGroup.setTag(rs.getString("tag")); + brokerGroup.setKind(BrokerGroupKind.fromCode(rs.getInt("kind"))); + + return brokerGroup; + }; + private static final RowMapper SUBJECT_ROUTE_ROW_MAPPER = (rs, rowNum) -> { + final String subject = rs.getString("subject_info"); + final String groupInfoJson = rs.getString("broker_group_json"); + final Timestamp updateTime = rs.getTimestamp("update_time"); + final int version = rs.getInt("version"); + final List groupNames = Serializer.deSerialize(groupInfoJson, new TypeReference>() { + }); + final SubjectRoute subjectRoute = new SubjectRoute(); + subjectRoute.setSubject(subject); + subjectRoute.setVersion(version); + subjectRoute.setBrokerGroups(groupNames); + subjectRoute.setUpdateTime(updateTime.getTime()); + return subjectRoute; + }; + private static final RowMapper SUBJECT_INFO_ROW_MAPPER = (rs, rowNum) -> { + final SubjectInfo subjectInfo = new SubjectInfo(); + subjectInfo.setName(rs.getString("name")); + subjectInfo.setTag(rs.getString("tag")); + subjectInfo.setUpdateTime(rs.getTimestamp("update_time").getTime()); + + return subjectInfo; + }; + + private final JdbcTemplate jdbcTemplate; + + public DatabaseStore() { + this.jdbcTemplate = JdbcTemplateHolder.getOrCreate(); + } + + @Override + public int insertSubjectRoute(String subject, int version, List groupNames) { + final String content = serialize(groupNames); + final Timestamp now = new Timestamp(System.currentTimeMillis()); + return jdbcTemplate.update(INSERT_SUBJECT_ROUTE_SQL, subject, version, content, now); + } + + @Override + public int updateSubjectRoute(String subject, int version, List groupNames) { + final String serialize = serialize(groupNames); + return jdbcTemplate.update(UPDATE_SUBJECT_ROUTE_SQL, serialize, subject, version); + } + + @Override + public SubjectRoute selectSubjectRoute(String subject) { + return jdbcTemplate.queryForObject(SELECT_SUBJECT_ROUTE_SQL, SUBJECT_ROUTE_ROW_MAPPER, subject); + } + + @Override + public void insertOrUpdateBrokerGroup(final String groupName, final BrokerGroupKind kind, final String masterAddress, final BrokerState brokerState) { + final Timestamp now = new Timestamp(System.currentTimeMillis()); + jdbcTemplate.update(INSERT_OR_UPDATE_BROKER_GROUP_SQL, groupName, kind.getCode(), masterAddress, brokerState.getCode(), now, masterAddress, brokerState.getCode()); + } + + @Override + public void updateBrokerGroup(String groupName, BrokerState brokerState) { + jdbcTemplate.update(UPDATE_BROKER_GROUP_SQL, brokerState.getCode(), groupName); + } + + @Override + public void updateBrokerGroupTag(String groupName, String tag) { + jdbcTemplate.update(UPDATE_BROKER_GROUP_TAG_SQL, tag, groupName); + } + + @Override + public void insertSubject(String subject, String tag) { + jdbcTemplate.update(INSERT_SUBJECT_INFO_SQL, subject, tag, new Date()); + } + + @Override + public List getAllBrokerGroups() { + return jdbcTemplate.query(FIND_BROKER_GROUP_SQL, BROKER_GROUP_ROW_MAPPER); + } + + @Override + public List getAllSubjectRoutes() { + try { + return jdbcTemplate.query(FIND_SUBJECT_ROUTE_SQL, SUBJECT_ROUTE_ROW_MAPPER); + } catch (Exception e) { + LOG.error("getAllSubjectRoutes error", e); + return Collections.emptyList(); + } + } + + @Override + public List getAllSubjectInfo() { + try { + return jdbcTemplate.query(ALL_SUBJECT_INFO_SQL, SUBJECT_INFO_ROW_MAPPER); + } catch (Exception e) { + LOG.error("getAllSubjectInfo error", e); + return Collections.emptyList(); + } + } + + @Override + public SubjectInfo getSubjectInfo(String subject) { + try { + return jdbcTemplate.queryForObject(QUERY_SUBJECT_INFO_SQL, SUBJECT_INFO_ROW_MAPPER, subject); + } catch (Exception e) { + LOG.error("getAllSubjectInfo error", e); + return null; + } + } + + @Override + public BrokerGroup getBrokerGroup(String groupName) { + try { + return jdbcTemplate.queryForObject(SELECT_BROKER_GROUP_SQL, BROKER_GROUP_ROW_MAPPER, groupName); + } catch (EmptyResultDataAccessException ignore) { + return null; + } + } + + @Override + public void insertClientMetaInfo(MetaInfoRequest request) { + final Timestamp now = new Timestamp(System.currentTimeMillis()); + jdbcTemplate.update(INSERT_CLIENT_META_INFO_SQL, request.getSubject(), request.getClientTypeCode(), + request.getConsumerGroup(), request.getClientId(), request.getAppCode(), now); + } +} diff --git a/qmq-metaserver/src/main/java/qunar/tc/qmq/meta/store/impl/Serializer.java b/qmq-metaserver/src/main/java/qunar/tc/qmq/meta/store/impl/Serializer.java new file mode 100644 index 00000000..23b26acc --- /dev/null +++ b/qmq-metaserver/src/main/java/qunar/tc/qmq/meta/store/impl/Serializer.java @@ -0,0 +1,57 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.meta.store.impl; + +import com.fasterxml.jackson.core.JsonParser; +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.DeserializationFeature; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.SerializationFeature; + +import java.io.IOException; + +class Serializer { + private static final ObjectMapper MAPPER = new ObjectMapper(); + + static { + MAPPER.disable(SerializationFeature.INDENT_OUTPUT); + MAPPER.disable(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES); + MAPPER.configure(JsonParser.Feature.ALLOW_BACKSLASH_ESCAPING_ANY_CHARACTER, true); + MAPPER.configure(JsonParser.Feature.ALLOW_COMMENTS, true); + MAPPER.configure(JsonParser.Feature.ALLOW_NON_NUMERIC_NUMBERS, true); + MAPPER.configure(JsonParser.Feature.ALLOW_NUMERIC_LEADING_ZEROS, true); + MAPPER.configure(JsonParser.Feature.ALLOW_SINGLE_QUOTES, true); + MAPPER.configure(JsonParser.Feature.ALLOW_UNQUOTED_CONTROL_CHARS, true); + MAPPER.configure(JsonParser.Feature.ALLOW_UNQUOTED_FIELD_NAMES, true); + } + + static String serialize(Object data) { + try { + return MAPPER.writeValueAsString(data); + } catch (IOException e) { + return null; + } + } + + static T deSerialize(String content, TypeReference typeReference) { + try { + return MAPPER.readValue(content, typeReference); + } catch (IOException e) { + return null; + } + } +} diff --git a/qmq-metaserver/src/main/java/qunar/tc/qmq/meta/utils/ClientLogUtils.java b/qmq-metaserver/src/main/java/qunar/tc/qmq/meta/utils/ClientLogUtils.java new file mode 100644 index 00000000..366ec61c --- /dev/null +++ b/qmq-metaserver/src/main/java/qunar/tc/qmq/meta/utils/ClientLogUtils.java @@ -0,0 +1,61 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.meta.utils; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import qunar.tc.qmq.configuration.DynamicConfig; +import qunar.tc.qmq.configuration.DynamicConfigLoader; + +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; + +/** + * @author keli.wang + * @since 2018/8/22 + */ +public final class ClientLogUtils { + private static final Logger LOG = LoggerFactory.getLogger(ClientLogUtils.class); + + private static final String DEFAULT_SWITCH = "default"; + + private static volatile Map logSwitchMap = Collections.emptyMap(); + private static volatile boolean defaultSwitch = true; + + static { + final DynamicConfig config = DynamicConfigLoader.load("client_log_switch.properties", false); + config.addListener(conf -> { + final Map tmp = new HashMap<>(); + conf.asMap().forEach((key, value) -> tmp.put(key, Boolean.parseBoolean(value))); + + defaultSwitch = tmp.getOrDefault(DEFAULT_SWITCH, true); + logSwitchMap = tmp; + }); + } + + private ClientLogUtils() { + } + + public static void log(final String subject, final String format, final Object... arguments) { + if (logSwitchMap.getOrDefault(subject, false)) { + LOG.info(format, arguments); + } else if (defaultSwitch) { + LOG.info(format, arguments); + } + } +} diff --git a/qmq-metaserver/src/main/java/qunar/tc/qmq/meta/utils/SubjectUtils.java b/qmq-metaserver/src/main/java/qunar/tc/qmq/meta/utils/SubjectUtils.java new file mode 100644 index 00000000..ebc46421 --- /dev/null +++ b/qmq-metaserver/src/main/java/qunar/tc/qmq/meta/utils/SubjectUtils.java @@ -0,0 +1,32 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.meta.utils; + +import com.google.common.base.Objects; + +/** + * @author keli.wang + * @since 2018/7/30 + */ +public final class SubjectUtils { + private SubjectUtils() { + } + + public static boolean isAnySubject(final String subject) { + return Objects.equal(subject, "*"); + } +} diff --git a/qmq-metaserver/src/main/java/qunar/tc/qmq/meta/web/JsonResult.java b/qmq-metaserver/src/main/java/qunar/tc/qmq/meta/web/JsonResult.java new file mode 100644 index 00000000..99a14cf4 --- /dev/null +++ b/qmq-metaserver/src/main/java/qunar/tc/qmq/meta/web/JsonResult.java @@ -0,0 +1,39 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.meta.web; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; + +/** + * @author keli.wang + * @since 2018-11-27 + */ +public class JsonResult { + public final int status; + public final String message; + public final T data; + + @JsonCreator + public JsonResult(@JsonProperty("status") int status, + @JsonProperty("message") String message, + @JsonProperty("data") T data) { + this.status = status; + this.message = message; + this.data = data; + } +} diff --git a/qmq-metaserver/src/main/java/qunar/tc/qmq/meta/web/MetaManagementServlet.java b/qmq-metaserver/src/main/java/qunar/tc/qmq/meta/web/MetaManagementServlet.java new file mode 100644 index 00000000..46b65b1e --- /dev/null +++ b/qmq-metaserver/src/main/java/qunar/tc/qmq/meta/web/MetaManagementServlet.java @@ -0,0 +1,59 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.meta.web; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.base.Strings; +import qunar.tc.qmq.meta.management.MetaManagementAction; +import qunar.tc.qmq.meta.management.MetaManagementActionSupplier; + +import javax.servlet.http.HttpServlet; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; +import java.io.IOException; + +/** + * @author keli.wang + * @since 2017/10/20 + */ +public class MetaManagementServlet extends HttpServlet { + private static final ObjectMapper MAPPER = new ObjectMapper(); + + private final MetaManagementActionSupplier actions = MetaManagementActionSupplier.getInstance(); + + @Override + protected void doPost(final HttpServletRequest req, final HttpServletResponse resp) throws IOException { + final String actionName = req.getParameter("action"); + if (Strings.isNullOrEmpty(actionName)) { + resp.setStatus(HttpServletResponse.SC_OK); + resp.getWriter().println("need provide action param"); + return; + } + + final MetaManagementAction action = actions.getAction(actionName); + if (action == null) { + resp.setStatus(HttpServletResponse.SC_OK); + resp.getWriter().println("不支持的 action: " + actionName); + return; + } + + final Object result = action.handleAction(req); + resp.setStatus(HttpServletResponse.SC_OK); + resp.setHeader("Content-Type", "application/json"); + resp.getWriter().println(MAPPER.writeValueAsString(result)); + } +} diff --git a/qmq-metaserver/src/main/java/qunar/tc/qmq/meta/web/MetaServerAddressSupplierServlet.java b/qmq-metaserver/src/main/java/qunar/tc/qmq/meta/web/MetaServerAddressSupplierServlet.java new file mode 100644 index 00000000..65163baf --- /dev/null +++ b/qmq-metaserver/src/main/java/qunar/tc/qmq/meta/web/MetaServerAddressSupplierServlet.java @@ -0,0 +1,49 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.meta.web; + +import com.google.common.base.Preconditions; +import com.google.common.base.Strings; +import qunar.tc.qmq.utils.NetworkUtils; + +import javax.servlet.http.HttpServlet; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; +import java.io.IOException; + +/** + * @author yunfeng.yang + * @since 2017/9/1 + */ +public class MetaServerAddressSupplierServlet extends HttpServlet { + private final String localhost; + + public MetaServerAddressSupplierServlet() { + this.localhost = NetworkUtils.getLocalAddress(); + Preconditions.checkArgument(!Strings.isNullOrEmpty(localhost), "get localhost error"); + } + + @Override + protected void doGet(HttpServletRequest req, HttpServletResponse resp) throws IOException { + resp.getWriter().write(buildLocalAddress()); + } + + private String buildLocalAddress() { + final int port = (int) getServletContext().getAttribute("port"); + return localhost + ":" + port; + } +} diff --git a/qmq-metaserver/src/main/java/qunar/tc/qmq/meta/web/OnOfflineServlet.java b/qmq-metaserver/src/main/java/qunar/tc/qmq/meta/web/OnOfflineServlet.java new file mode 100644 index 00000000..5c7823ca --- /dev/null +++ b/qmq-metaserver/src/main/java/qunar/tc/qmq/meta/web/OnOfflineServlet.java @@ -0,0 +1,90 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.meta.web; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.base.Strings; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import qunar.tc.qmq.base.OnOfflineState; +import qunar.tc.qmq.meta.cache.CachedOfflineStateManager; +import qunar.tc.qmq.meta.model.ClientOfflineState; + +import javax.servlet.ServletException; +import javax.servlet.http.HttpServlet; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; +import java.io.IOException; +import java.io.PrintWriter; + +/** + * yiqun.fan@qunar.com 2018/3/7 + */ +public class OnOfflineServlet extends HttpServlet { + private static final Logger LOGGER = LoggerFactory.getLogger(OnOfflineServlet.class); + + private static final ObjectMapper MAPPER = new ObjectMapper(); + + private final CachedOfflineStateManager offlineStateManager = CachedOfflineStateManager.SUPPLIER.get(); + + @Override + protected void doGet(HttpServletRequest req, HttpServletResponse resp) throws IOException { + final String operation = req.getParameter("operation"); + final String clientId = req.getParameter("clientId"); + final String subject = req.getParameter("subject"); + final String consumerGroup = req.getParameter("consumerGroup"); + final OnOfflineState state = getOnOfflineState(req.getParameter("state")); + + resp.setStatus(HttpServletResponse.SC_OK); + resp.setHeader("Content-Type", "application/json"); + PrintWriter out = resp.getWriter(); + + if (Strings.isNullOrEmpty(operation) || Strings.isNullOrEmpty(clientId) || Strings.isNullOrEmpty(subject) || Strings.isNullOrEmpty(consumerGroup) || state == null) { + out.println(MAPPER.writeValueAsString(new JsonResult<>(ResultStatus.SYSTEM_ERROR, "invalid parameter", null))); + return; + } + + ClientOfflineState clientState = new ClientOfflineState(); + clientState.setClientId(clientId); + clientState.setSubject(subject); + clientState.setConsumerGroup(consumerGroup); + clientState.setState(state); + + try { + if ("update".equals(operation)) { + offlineStateManager.insertOrUpdate(clientState); + out.println(MAPPER.writeValueAsString(new JsonResult(ResultStatus.OK, "ok", null))); + } else if ("query".equals(operation)) { + offlineStateManager.queryClientState(clientState); + out.println(MAPPER.writeValueAsString(new JsonResult(ResultStatus.OK, clientState.getState().toString(), null))); + } else { + out.println(MAPPER.writeValueAsString(new JsonResult(ResultStatus.SYSTEM_ERROR, "unsupport: " + operation, null))); + } + } catch (Exception e) { + LOGGER.error("onoffline exception. {}", clientState, e); + out.println(MAPPER.writeValueAsString(new JsonResult(ResultStatus.SYSTEM_ERROR, e.getClass().getCanonicalName() + ": " + e.getMessage(), null))); + } + } + + private OnOfflineState getOnOfflineState(String state) { + try { + return OnOfflineState.fromCode(Integer.parseInt(state)); + } catch (Exception e) { + return null; + } + } +} diff --git a/qmq-metaserver/src/main/java/qunar/tc/qmq/meta/web/ResultStatus.java b/qmq-metaserver/src/main/java/qunar/tc/qmq/meta/web/ResultStatus.java new file mode 100644 index 00000000..d86b8ada --- /dev/null +++ b/qmq-metaserver/src/main/java/qunar/tc/qmq/meta/web/ResultStatus.java @@ -0,0 +1,26 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.meta.web; + +/** + * @author keli.wang + * @since 2018-11-27 + */ +public final class ResultStatus { + public static final int OK = 0; + public static final int SYSTEM_ERROR = -1; +} diff --git a/qmq-metaserver/src/main/java/qunar/tc/qmq/meta/web/SubjectConsumerServlet.java b/qmq-metaserver/src/main/java/qunar/tc/qmq/meta/web/SubjectConsumerServlet.java new file mode 100644 index 00000000..b1873d4a --- /dev/null +++ b/qmq-metaserver/src/main/java/qunar/tc/qmq/meta/web/SubjectConsumerServlet.java @@ -0,0 +1,67 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.meta.web; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.base.Strings; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import qunar.tc.qmq.meta.model.GroupedConsumer; +import qunar.tc.qmq.meta.route.SubjectConsumerService; +import qunar.tc.qmq.meta.route.impl.SubjectConsumerServiceImpl; + +import javax.servlet.ServletException; +import javax.servlet.http.HttpServlet; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; +import java.io.IOException; +import java.util.List; + +/** + * @author keli.wang + * @since 2017/12/5 + */ +public class SubjectConsumerServlet extends HttpServlet { + private static final Logger LOG = LoggerFactory.getLogger(SubjectConsumerServlet.class); + + private static final ObjectMapper MAPPER = new ObjectMapper(); + + private final SubjectConsumerService subjectConsumerService = new SubjectConsumerServiceImpl(); + + @Override + protected void doGet(HttpServletRequest req, HttpServletResponse resp) throws ServletException, IOException { + final String subject = req.getParameter("subject"); + + resp.setStatus(HttpServletResponse.SC_OK); + resp.setHeader("Content-Type", "application/json"); + + if (Strings.isNullOrEmpty(subject)) { + resp.getWriter().println(MAPPER.writeValueAsString(new JsonResult<>(ResultStatus.SYSTEM_ERROR, "请求的主题不能为空", null))); + return; + } + + try { + final List consumers = subjectConsumerService.consumers(subject); + resp.getWriter().println(MAPPER.writeValueAsString(new JsonResult<>(ResultStatus.OK, "成功", consumers))); + return; + } catch (Exception e) { + LOG.error("Failed get subject consumers. subject: {}", subject, e); + } + + resp.getWriter().println(MAPPER.writeValueAsString(new JsonResult(ResultStatus.SYSTEM_ERROR, "未知错误", null))); + } +} diff --git a/qmq-metrics-prometheus/pom.xml b/qmq-metrics-prometheus/pom.xml new file mode 100644 index 00000000..68f74641 --- /dev/null +++ b/qmq-metrics-prometheus/pom.xml @@ -0,0 +1,25 @@ + + + 4.0.0 + + + qmq + qunar.tc + 4.0.30 + + + qmq-metrics-prometheus + + + + io.prometheus + simpleclient + + + qunar.tc + qmq-common + + + \ No newline at end of file diff --git a/qmq-metrics-prometheus/src/main/java/qunar/tc/qmq/metrics/prometheus/PrometheusQmqCounter.java b/qmq-metrics-prometheus/src/main/java/qunar/tc/qmq/metrics/prometheus/PrometheusQmqCounter.java new file mode 100644 index 00000000..d1764be5 --- /dev/null +++ b/qmq-metrics-prometheus/src/main/java/qunar/tc/qmq/metrics/prometheus/PrometheusQmqCounter.java @@ -0,0 +1,52 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.metrics.prometheus; + +import io.prometheus.client.Gauge; +import qunar.tc.qmq.metrics.QmqCounter; + +/** + * @author keli.wang + * @since 2018/11/22 + */ +public class PrometheusQmqCounter implements QmqCounter { + private final Gauge.Child gauge; + + public PrometheusQmqCounter(final Gauge gauge, final String[] labels) { + this.gauge = gauge.labels(labels); + } + + @Override + public void inc() { + gauge.inc(); + } + + @Override + public void inc(final long n) { + gauge.inc(n); + } + + @Override + public void dec() { + gauge.dec(); + } + + @Override + public void dec(final long n) { + gauge.dec(n); + } +} diff --git a/qmq-metrics-prometheus/src/main/java/qunar/tc/qmq/metrics/prometheus/PrometheusQmqGauge.java b/qmq-metrics-prometheus/src/main/java/qunar/tc/qmq/metrics/prometheus/PrometheusQmqGauge.java new file mode 100644 index 00000000..1dbcc063 --- /dev/null +++ b/qmq-metrics-prometheus/src/main/java/qunar/tc/qmq/metrics/prometheus/PrometheusQmqGauge.java @@ -0,0 +1,81 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.metrics.prometheus; + +import com.google.common.base.Supplier; +import io.prometheus.client.Collector; +import io.prometheus.client.GaugeMetricFamily; +import io.prometheus.client.SimpleCollector; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Map; + +public class PrometheusQmqGauge extends SimpleCollector implements Collector.Describable { + + PrometheusQmqGauge(Builder b) { + super(b); + } + + public static Builder build() { + return new Builder(); + } + + @Override + protected Child newChild() { + return new Child(); + } + + @Override + public List collect() { + List samples = new ArrayList<>(children.size()); + for (Map.Entry, Child> c : children.entrySet()) { + samples.add(new MetricFamilySamples.Sample(fullname, labelNames, c.getKey(), c.getValue().get())); + } + return familySamplesList(Type.GAUGE, samples); + } + + @Override + public List describe() { + return Collections.singletonList(new GaugeMetricFamily(fullname, help, labelNames)); + } + + public static class Builder extends SimpleCollector.Builder { + @Override + public PrometheusQmqGauge create() { + return new PrometheusQmqGauge(this); + } + } + + public static class Child { + private Supplier supplier; + + public void setSupplier(final Supplier supplier) { + this.supplier = supplier; + } + + public double get() { + if (supplier == null) { + return 0; + } else { + return supplier.get(); + } + } + } +} + diff --git a/qmq-metrics-prometheus/src/main/java/qunar/tc/qmq/metrics/prometheus/PrometheusQmqMeter.java b/qmq-metrics-prometheus/src/main/java/qunar/tc/qmq/metrics/prometheus/PrometheusQmqMeter.java new file mode 100644 index 00000000..0a566c76 --- /dev/null +++ b/qmq-metrics-prometheus/src/main/java/qunar/tc/qmq/metrics/prometheus/PrometheusQmqMeter.java @@ -0,0 +1,42 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.metrics.prometheus; + +import io.prometheus.client.Summary; +import qunar.tc.qmq.metrics.QmqMeter; + +/** + * @author keli.wang + * @since 2018/11/22 + */ +public class PrometheusQmqMeter implements QmqMeter { + private final Summary.Child summary; + + public PrometheusQmqMeter(final Summary summary, final String[] labels) { + this.summary = summary.labels(labels); + } + + @Override + public void mark() { + summary.observe(1); + } + + @Override + public void mark(final long n) { + summary.observe(n); + } +} diff --git a/qmq-metrics-prometheus/src/main/java/qunar/tc/qmq/metrics/prometheus/PrometheusQmqMetricRegistry.java b/qmq-metrics-prometheus/src/main/java/qunar/tc/qmq/metrics/prometheus/PrometheusQmqMetricRegistry.java new file mode 100644 index 00000000..7fb81c38 --- /dev/null +++ b/qmq-metrics-prometheus/src/main/java/qunar/tc/qmq/metrics/prometheus/PrometheusQmqMetricRegistry.java @@ -0,0 +1,180 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.metrics.prometheus; + +import com.google.common.base.Supplier; +import com.google.common.cache.CacheBuilder; +import com.google.common.cache.CacheLoader; +import com.google.common.cache.LoadingCache; +import io.prometheus.client.Collector; +import io.prometheus.client.Gauge; +import io.prometheus.client.SimpleCollector; +import io.prometheus.client.Summary; +import qunar.tc.qmq.metrics.QmqCounter; +import qunar.tc.qmq.metrics.QmqMeter; +import qunar.tc.qmq.metrics.QmqMetricRegistry; +import qunar.tc.qmq.metrics.QmqTimer; + +import java.util.Arrays; + +/** + * @author keli.wang + * @since 2018/11/22 + */ +public class PrometheusQmqMetricRegistry implements QmqMetricRegistry { + private static final LoadingCache CACHE = CacheBuilder.newBuilder() + .build(new CacheLoader() { + @Override + public Collector load(Key key) { + return key.create(); + } + }); + + @SuppressWarnings("unchecked") + private static M cacheFor(Key key) { + return (M) CACHE.getUnchecked(key); + } + + @Override + public void newGauge(final String name, final String[] tags, final String[] values, final Supplier supplier) { + final PrometheusQmqGauge gauge = cacheFor(new GuageKey(name, tags)); + gauge.labels(values).setSupplier(supplier); + } + + @Override + public QmqCounter newCounter(final String name, final String[] tags, final String[] values) { + final Gauge gauge = cacheFor(new CounterKey(name, tags)); + return new PrometheusQmqCounter(gauge, values); + } + + @Override + public QmqMeter newMeter(final String name, final String[] tags, final String[] values) { + final Summary summary = cacheFor(new MeterKey(name, tags)); + return new PrometheusQmqMeter(summary, values); + } + + @Override + public QmqTimer newTimer(final String name, final String[] tags, final String[] values) { + final Summary summary = cacheFor(new TimerKey(name, tags)); + return new PrometheusQmqTimer(summary, values); + } + + @Override + public void remove(final String name, final String[] tags, final String[] values) { + // TODO(keli.wang): only remove child collectors for now, may we should remove whole metric in the future + final SimpleCollector collector = cacheFor(new SimpleCollectorKey(name, tags)); + collector.remove(values); + } + + private static abstract class Key { + final String name; + final String[] tags; + + Key(String name, String[] tags) { + this.name = name; + this.tags = tags; + } + + @Override + public boolean equals(Object o) { + if (this == o) + return true; + if (o == null || getClass() != o.getClass()) + return false; + + Key key = (Key) o; + + if (name != null ? !name.equals(key.name) : key.name != null) + return false; + return Arrays.equals(tags, key.tags); + } + + @Override + public int hashCode() { + int result = name != null ? name.hashCode() : 0; + result = 31 * result + (tags != null ? Arrays.hashCode(tags) : 0); + return result; + } + + public abstract M create(); + } + + private static class SimpleCollectorKey extends Key { + + SimpleCollectorKey(final String name, final String[] tags) { + super(name, tags); + } + + @Override + public SimpleCollector create() { + throw new UnsupportedOperationException("cannot create simple collector"); + } + } + + private static class GuageKey extends Key { + GuageKey(final String name, final String[] tags) { + super(name, tags); + } + + @Override + public PrometheusQmqGauge create() { + return PrometheusQmqGauge.build().name(name).labelNames(tags).create().register(); + } + } + + private static class CounterKey extends Key { + CounterKey(final String name, final String[] tags) { + super(name, tags); + } + + @Override + public Gauge create() { + return Gauge.build().name(name).labelNames(tags).create().register(); + } + } + + private static class MeterKey extends Key

{ + + MeterKey(final String name, final String[] tags) { + super(name, tags); + } + + @Override + public Summary create() { + return Summary.build().name(name).labelNames(tags).create().register(); + } + } + + private static class TimerKey extends Key { + + TimerKey(final String name, final String[] tags) { + super(name, tags); + } + + @Override + public Summary create() { + return Summary.build() + .name(name) + .labelNames(tags) + .quantile(0.5, 0.05) + .quantile(0.75, 0.05) + .quantile(0.99, 0.05) + .create() + .register(); + } + } +} diff --git a/qmq-metrics-prometheus/src/main/java/qunar/tc/qmq/metrics/prometheus/PrometheusQmqTimer.java b/qmq-metrics-prometheus/src/main/java/qunar/tc/qmq/metrics/prometheus/PrometheusQmqTimer.java new file mode 100644 index 00000000..b09b424c --- /dev/null +++ b/qmq-metrics-prometheus/src/main/java/qunar/tc/qmq/metrics/prometheus/PrometheusQmqTimer.java @@ -0,0 +1,39 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.metrics.prometheus; + +import io.prometheus.client.Summary; +import qunar.tc.qmq.metrics.QmqTimer; + +import java.util.concurrent.TimeUnit; + +/** + * @author keli.wang + * @since 2018/11/22 + */ +public class PrometheusQmqTimer implements QmqTimer { + private final Summary.Child summary; + + public PrometheusQmqTimer(final Summary summary, final String[] labels) { + this.summary = summary.labels(labels); + } + + @Override + public void update(final long duration, final TimeUnit unit) { + summary.observe(unit.toMillis(duration)); + } +} diff --git a/qmq-metrics-prometheus/src/main/resources/META-INF/services/qunar.tc.qmq.metrics.QmqMetricRegistry b/qmq-metrics-prometheus/src/main/resources/META-INF/services/qunar.tc.qmq.metrics.QmqMetricRegistry new file mode 100644 index 00000000..c7d6c24e --- /dev/null +++ b/qmq-metrics-prometheus/src/main/resources/META-INF/services/qunar.tc.qmq.metrics.QmqMetricRegistry @@ -0,0 +1 @@ +qunar.tc.qmq.metrics.prometheus.PrometheusQmqMetricRegistry \ No newline at end of file diff --git a/qmq-remoting/pom.xml b/qmq-remoting/pom.xml new file mode 100644 index 00000000..e4de5567 --- /dev/null +++ b/qmq-remoting/pom.xml @@ -0,0 +1,40 @@ + + + 4.0.0 + + + qmq + qunar.tc + 4.0.30 + + + qmq-remoting + + + 1.7 + 1.7 + + + + + ${project.groupId} + qmq-common + + + io.netty + netty-all + + + com.google.guava + guava + + + + junit + junit + + + + \ No newline at end of file diff --git a/qmq-remoting/src/main/java/qunar/tc/qmq/netty/DecodeHandler.java b/qmq-remoting/src/main/java/qunar/tc/qmq/netty/DecodeHandler.java new file mode 100644 index 00000000..81220d90 --- /dev/null +++ b/qmq-remoting/src/main/java/qunar/tc/qmq/netty/DecodeHandler.java @@ -0,0 +1,96 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.netty; + +import io.netty.buffer.ByteBuf; +import io.netty.buffer.Unpooled; +import io.netty.channel.ChannelHandlerContext; +import io.netty.handler.codec.ByteToMessageDecoder; +import qunar.tc.qmq.protocol.RemotingCommand; +import qunar.tc.qmq.protocol.RemotingHeader; + +import java.io.IOException; +import java.util.List; + +import static qunar.tc.qmq.protocol.RemotingHeader.DEFAULT_MAGIC_CODE; +import static qunar.tc.qmq.protocol.RemotingHeader.VERSION_3; + +/** + * @author yunfeng.yang + * @since 2017/6/30 + */ +public class DecodeHandler extends ByteToMessageDecoder { + private final boolean isServer; + + public DecodeHandler(boolean isServer) { + this.isServer = isServer; + } + + @Override + protected void decode(ChannelHandlerContext channelHandlerContext, ByteBuf in, List list) throws Exception { + if (in.readableBytes() < RemotingHeader.MIN_HEADER_SIZE + RemotingHeader.LENGTH_FIELD) return; + + int magicCode = in.getInt(in.readerIndex() + RemotingHeader.LENGTH_FIELD); + if (DEFAULT_MAGIC_CODE != magicCode) { + throw new IOException("非法数据,MagicCode=" + Integer.toHexString(magicCode)); + } + + in.markReaderIndex(); + int total = in.readInt(); + if (in.readableBytes() < total) { + in.resetReaderIndex(); + return; + } + + short headerSize = in.readShort(); + RemotingHeader remotingHeader = decodeHeader(in); + + int bodyLength = total - headerSize - RemotingHeader.HEADER_SIZE_LEN; + + RemotingCommand remotingCommand = new RemotingCommand(); + if (isServer) { + ByteBuf bodyData = in.readSlice(bodyLength); + bodyData.retain(); + remotingCommand.setBody(bodyData); + } else { + ByteBuf bodyData = Unpooled.buffer(bodyLength, bodyLength); + in.readBytes(bodyData, bodyLength); + remotingCommand.setBody(bodyData); + } + remotingCommand.setHeader(remotingHeader); + list.add(remotingCommand); + } + + private RemotingHeader decodeHeader(ByteBuf in) { + RemotingHeader remotingHeader = new RemotingHeader(); + // int magicCode(4 bytes) + remotingHeader.setMagicCode(in.readInt()); + // short code + remotingHeader.setCode(in.readShort()); + // short version + remotingHeader.setVersion(in.readShort()); + // int opaque + remotingHeader.setOpaque(in.readInt()); + // int flag + remotingHeader.setFlag(in.readInt()); + // int requestCode if version == 3 + if (remotingHeader.getVersion() >= VERSION_3) { + remotingHeader.setRequestCode(in.readShort()); + } + return remotingHeader; + } +} diff --git a/qmq-remoting/src/main/java/qunar/tc/qmq/netty/EncodeHandler.java b/qmq-remoting/src/main/java/qunar/tc/qmq/netty/EncodeHandler.java new file mode 100644 index 00000000..83d5c644 --- /dev/null +++ b/qmq-remoting/src/main/java/qunar/tc/qmq/netty/EncodeHandler.java @@ -0,0 +1,65 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.netty; + +import io.netty.buffer.ByteBuf; +import io.netty.channel.ChannelHandlerContext; +import io.netty.handler.codec.MessageToByteEncoder; +import qunar.tc.qmq.protocol.Datagram; +import qunar.tc.qmq.protocol.RemotingHeader; + +/** + * @author yunfeng.yang + * @since 2017/6/30 + */ +public class EncodeHandler extends MessageToByteEncoder { + + //total|header size|header|body + //total = len(header size) + len(header) + len(body) + //header size = len(header) + @Override + protected void encode(ChannelHandlerContext ctx, Datagram msg, ByteBuf out) throws Exception { + int start = out.writerIndex(); + int headerStart = start + RemotingHeader.LENGTH_FIELD; + out.ensureWritable(RemotingHeader.LENGTH_FIELD); + out.writerIndex(headerStart); + + final RemotingHeader header = msg.getHeader(); + encodeHeader(header, out); + int headerSize = out.writerIndex() - headerStart; + + msg.writeBody(out); + int end = out.writerIndex(); + int total = end - start - RemotingHeader.TOTAL_SIZE_LEN; + + out.writerIndex(start); + out.writeInt(total); + out.writeShort((short) headerSize); + out.writerIndex(end); + } + + private static void encodeHeader(final RemotingHeader header, final ByteBuf out) { + out.writeInt(header.getMagicCode()); + out.writeShort(header.getCode()); + out.writeShort(header.getVersion()); + out.writeInt(header.getOpaque()); + out.writeInt(header.getFlag()); + if (header.getVersion() >= RemotingHeader.VERSION_3) { + out.writeShort(header.getRequestCode()); + } + } +} diff --git a/qmq-remoting/src/main/java/qunar/tc/qmq/netty/NettyClientConfig.java b/qmq-remoting/src/main/java/qunar/tc/qmq/netty/NettyClientConfig.java new file mode 100644 index 00000000..510f49c1 --- /dev/null +++ b/qmq-remoting/src/main/java/qunar/tc/qmq/netty/NettyClientConfig.java @@ -0,0 +1,77 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.netty; + +/** + * @author yiqun.fan create on 17-7-3. + */ +public class NettyClientConfig { + private int clientWorkerThreads = Math.min(4, Runtime.getRuntime().availableProcessors() * 2); + private int connectTimeoutMillis = 5000; + private long channelNotActiveInterval = 1000 * 60; + private int clientChannelMaxIdleTimeSeconds = 120; + private int clientSocketSndBufSize = Integer.getInteger("qmq.remoting.socket.sndbuf.size", 65535); + private int clientSocketRcvBufSize = Integer.getInteger("qmq.remoting.socket.rcvbuf.size", 65535); + + public int getClientWorkerThreads() { + return clientWorkerThreads; + } + + public void setClientWorkerThreads(int clientWorkerThreads) { + this.clientWorkerThreads = clientWorkerThreads; + } + + public int getConnectTimeoutMillis() { + return connectTimeoutMillis; + } + + public void setConnectTimeoutMillis(int connectTimeoutMillis) { + this.connectTimeoutMillis = connectTimeoutMillis; + } + + public long getChannelNotActiveInterval() { + return channelNotActiveInterval; + } + + public void setChannelNotActiveInterval(long channelNotActiveInterval) { + this.channelNotActiveInterval = channelNotActiveInterval; + } + + public int getClientChannelMaxIdleTimeSeconds() { + return clientChannelMaxIdleTimeSeconds; + } + + public void setClientChannelMaxIdleTimeSeconds(int clientChannelMaxIdleTimeSeconds) { + this.clientChannelMaxIdleTimeSeconds = clientChannelMaxIdleTimeSeconds; + } + + public int getClientSocketSndBufSize() { + return clientSocketSndBufSize; + } + + public void setClientSocketSndBufSize(int clientSocketSndBufSize) { + this.clientSocketSndBufSize = clientSocketSndBufSize; + } + + public int getClientSocketRcvBufSize() { + return clientSocketRcvBufSize; + } + + public void setClientSocketRcvBufSize(int clientSocketRcvBufSize) { + this.clientSocketRcvBufSize = clientSocketRcvBufSize; + } +} diff --git a/qmq-remoting/src/main/java/qunar/tc/qmq/netty/client/AbstractNettyClient.java b/qmq-remoting/src/main/java/qunar/tc/qmq/netty/client/AbstractNettyClient.java new file mode 100644 index 00000000..4682e36e --- /dev/null +++ b/qmq-remoting/src/main/java/qunar/tc/qmq/netty/client/AbstractNettyClient.java @@ -0,0 +1,102 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.netty.client; + +import io.netty.bootstrap.Bootstrap; +import io.netty.channel.Channel; +import io.netty.channel.ChannelInitializer; +import io.netty.channel.ChannelOption; +import io.netty.channel.EventLoopGroup; +import io.netty.channel.nio.NioEventLoopGroup; +import io.netty.channel.socket.SocketChannel; +import io.netty.channel.socket.nio.NioSocketChannel; +import io.netty.util.concurrent.DefaultEventExecutorGroup; +import io.netty.util.concurrent.DefaultThreadFactory; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import qunar.tc.qmq.netty.NettyClientConfig; +import qunar.tc.qmq.netty.exception.ClientSendException; + +import java.util.concurrent.atomic.AtomicBoolean; + +/** + * @author yiqun.fan create on 17-8-31. + */ +public abstract class AbstractNettyClient { + private static final Logger LOGGER = LoggerFactory.getLogger(AbstractNettyClient.class); + + private final String clientName; + private final AtomicBoolean started = new AtomicBoolean(false); + private EventLoopGroup eventLoopGroup; + private DefaultEventExecutorGroup eventExecutors; + private NettyConnectManageHandler connectManager; + + protected AbstractNettyClient(String clientName) { + this.clientName = clientName; + } + + public boolean isStarted() { + return started.get(); + } + + public synchronized void start(NettyClientConfig config) { + if (started.get()) { + return; + } + initHandler(); + Bootstrap bootstrap = new Bootstrap(); + eventLoopGroup = new NioEventLoopGroup(1, new DefaultThreadFactory(clientName + "-boss")); + eventExecutors = new DefaultEventExecutorGroup(config.getClientWorkerThreads(), new DefaultThreadFactory(clientName + "-worker")); + connectManager = new NettyConnectManageHandler(bootstrap, config.getConnectTimeoutMillis()); + bootstrap.group(this.eventLoopGroup) + .channel(NioSocketChannel.class) + .option(ChannelOption.TCP_NODELAY, true) + .option(ChannelOption.SO_KEEPALIVE, false) + .option(ChannelOption.CONNECT_TIMEOUT_MILLIS, config.getConnectTimeoutMillis()) + .option(ChannelOption.SO_SNDBUF, config.getClientSocketSndBufSize()) + .option(ChannelOption.SO_RCVBUF, config.getClientSocketRcvBufSize()) + .handler(newChannelInitializer(config, eventExecutors, connectManager)); + started.set(true); + } + + public synchronized void shutdown() { + if (!started.get()) { + return; + } + try { + connectManager.shutdown(); + eventLoopGroup.shutdownGracefully(); + eventExecutors.shutdownGracefully(); + destroyHandler(); + started.set(false); + } catch (Exception e) { + LOGGER.error("NettyClient {} shutdown exception, ", clientName, e); + } + } + + protected void initHandler() { + } + + protected void destroyHandler() { + } + + protected abstract ChannelInitializer newChannelInitializer(NettyClientConfig config, DefaultEventExecutorGroup eventExecutors, NettyConnectManageHandler connectManager); + + protected Channel getOrCreateChannel(String remoteAddr) throws ClientSendException { + return connectManager.getOrCreateChannel(remoteAddr); + } +} diff --git a/qmq-remoting/src/main/java/qunar/tc/qmq/netty/client/ChannelWrapper.java b/qmq-remoting/src/main/java/qunar/tc/qmq/netty/client/ChannelWrapper.java new file mode 100644 index 00000000..b600de31 --- /dev/null +++ b/qmq-remoting/src/main/java/qunar/tc/qmq/netty/client/ChannelWrapper.java @@ -0,0 +1,43 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.netty.client; + +import io.netty.channel.Channel; +import io.netty.channel.ChannelFuture; + +/** + * @author yiqun.fan create on 17-8-29. + */ +class ChannelWrapper { + private final ChannelFuture channelFuture; + + ChannelWrapper(ChannelFuture channelFuture) { + this.channelFuture = channelFuture; + } + + ChannelFuture getChannelFuture() { + return channelFuture; + } + + Channel getChannel() { + return this.channelFuture.channel(); + } + + boolean isOK() { + return this.channelFuture.channel() != null && this.channelFuture.channel().isActive(); + } +} diff --git a/qmq-remoting/src/main/java/qunar/tc/qmq/netty/client/HttpResponseCallback.java b/qmq-remoting/src/main/java/qunar/tc/qmq/netty/client/HttpResponseCallback.java new file mode 100644 index 00000000..5e18d8c9 --- /dev/null +++ b/qmq-remoting/src/main/java/qunar/tc/qmq/netty/client/HttpResponseCallback.java @@ -0,0 +1,26 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.netty.client; + +/** + * @author yiqun.fan create on 17-8-29. + */ +public interface HttpResponseCallback { + V onCompleted(Response response) throws Exception; + + void onThrowable(Throwable t); +} diff --git a/qmq-remoting/src/main/java/qunar/tc/qmq/netty/client/NettyClient.java b/qmq-remoting/src/main/java/qunar/tc/qmq/netty/client/NettyClient.java new file mode 100644 index 00000000..01ef884a --- /dev/null +++ b/qmq-remoting/src/main/java/qunar/tc/qmq/netty/client/NettyClient.java @@ -0,0 +1,159 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.netty.client; + +import com.google.common.util.concurrent.AbstractFuture; +import io.netty.channel.Channel; +import io.netty.channel.ChannelFuture; +import io.netty.channel.ChannelFutureListener; +import io.netty.channel.ChannelInitializer; +import io.netty.channel.socket.SocketChannel; +import io.netty.handler.timeout.IdleStateHandler; +import io.netty.util.concurrent.DefaultEventExecutorGroup; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import qunar.tc.qmq.netty.DecodeHandler; +import qunar.tc.qmq.netty.EncodeHandler; +import qunar.tc.qmq.netty.NettyClientConfig; +import qunar.tc.qmq.netty.exception.ClientSendException; +import qunar.tc.qmq.netty.exception.RemoteTimeoutException; +import qunar.tc.qmq.protocol.Datagram; +import qunar.tc.qmq.util.RemoteHelper; + +import java.util.concurrent.ExecutionException; + +import static qunar.tc.qmq.netty.exception.ClientSendException.SendErrorCode; + +/** + * @author yiqun.fan create on 17-7-3. + */ +public class NettyClient extends AbstractNettyClient { + private static final Logger LOGGER = LoggerFactory.getLogger(NettyClient.class); + private static final NettyClient INSTANCE = new NettyClient(); + + public static NettyClient getClient() { + return INSTANCE; + } + + private NettyClientHandler clientHandler; + + private NettyClient() { + super("qmq-client"); + } + + @Override + protected void initHandler() { + clientHandler = new NettyClientHandler(); + } + + @Override + protected void destroyHandler() { + clientHandler.shutdown(); + } + + @Override + protected ChannelInitializer newChannelInitializer(final NettyClientConfig config, final DefaultEventExecutorGroup eventExecutors, final NettyConnectManageHandler connectManager) { + return new ChannelInitializer() { + @Override + protected void initChannel(SocketChannel ch) throws Exception { + ch.pipeline().addLast(eventExecutors, + new EncodeHandler(), + new DecodeHandler(false), + new IdleStateHandler(0, 0, config.getClientChannelMaxIdleTimeSeconds()), + connectManager, + clientHandler); + } + }; + } + + public Datagram sendSync(String brokerAddr, Datagram request, long responseTimeout) throws ClientSendException, InterruptedException, RemoteTimeoutException { + ResultFuture result = new ResultFuture(brokerAddr); + sendAsync(brokerAddr, request, responseTimeout, result); + try { + return result.get(); + } catch (ExecutionException e) { + if (e.getCause() instanceof RemoteTimeoutException) { + throw (RemoteTimeoutException) e.getCause(); + } + + if (e.getCause() instanceof ClientSendException) { + throw (ClientSendException) e.getCause(); + } + + throw new RuntimeException(e.getCause()); + } + } + + private static final class ResultFuture extends AbstractFuture implements ResponseFuture.Callback { + private final String brokerAddr; + + ResultFuture(String brokerAddr) { + this.brokerAddr = brokerAddr; + } + + @Override + public void processResponse(ResponseFuture responseFuture) { + if (!responseFuture.isSendOk()) { + setException(new ClientSendException(ClientSendException.SendErrorCode.WRITE_CHANNEL_FAIL)); + return; + } + + if (responseFuture.isTimeout()) { + setException(new RemoteTimeoutException(brokerAddr, responseFuture.getTimeout())); + return; + } + + Datagram response = responseFuture.getResponse(); + if (response != null) { + set(response); + } else { + setException(new ClientSendException(SendErrorCode.BROKER_BUSY)); + } + } + } + + public void sendAsync(String brokerAddr, Datagram request, long responseTimeout, ResponseFuture.Callback callback) throws ClientSendException { + final Channel channel = getOrCreateChannel(brokerAddr); + final ResponseFuture responseFuture = clientHandler.newResponse(channel, responseTimeout, callback); + request.getHeader().setOpaque(responseFuture.getOpaque()); + + try { + channel.writeAndFlush(request).addListener(new ChannelFutureListener() { + @Override + public void operationComplete(ChannelFuture future) { + if (future.isSuccess()) { + responseFuture.setSendOk(true); + return; + } + clientHandler.removeResponse(channel, responseFuture); + responseFuture.completeBySendFail(future.cause()); + LOGGER.error("send request to broker failed.", future.cause()); + try { + responseFuture.executeCallbackOnlyOnce(); + } catch (Throwable e) { + LOGGER.error("execute callback when send error exception", e); + } + } + }); + } catch (Exception e) { + clientHandler.removeResponse(channel, responseFuture); + responseFuture.completeBySendFail(e); + LOGGER.warn("send request fail. brokerAddr={}", brokerAddr); + throw new ClientSendException(SendErrorCode.WRITE_CHANNEL_FAIL, RemoteHelper.parseChannelRemoteAddress(channel), e); + } + } +} diff --git a/qmq-remoting/src/main/java/qunar/tc/qmq/netty/client/NettyClientHandler.java b/qmq-remoting/src/main/java/qunar/tc/qmq/netty/client/NettyClientHandler.java new file mode 100644 index 00000000..2d9861f9 --- /dev/null +++ b/qmq-remoting/src/main/java/qunar/tc/qmq/netty/client/NettyClientHandler.java @@ -0,0 +1,163 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.netty.client; + +import io.netty.channel.Channel; +import io.netty.channel.ChannelHandler; +import io.netty.channel.ChannelHandlerContext; +import io.netty.channel.SimpleChannelInboundHandler; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import qunar.tc.qmq.concurrent.NamedThreadFactory; +import qunar.tc.qmq.netty.exception.ClientSendException; +import qunar.tc.qmq.protocol.Datagram; +import qunar.tc.qmq.util.RemoteHelper; + +import java.util.Iterator; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.concurrent.*; +import java.util.concurrent.atomic.AtomicInteger; + +/** + * @author yiqun.fan create on 17-8-29. + */ +@ChannelHandler.Sharable +class NettyClientHandler extends SimpleChannelInboundHandler { + private static final Logger LOGGER = LoggerFactory.getLogger(NettyClientHandler.class); + + private static final long CLEAN_RESPONSE_TABLE_PERIOD_MILLIS = 1000; + + private final AtomicInteger opaque = new AtomicInteger(0); + private final ConcurrentMap> requestsInFlight = new ConcurrentHashMap<>(4); + private final ScheduledExecutorService timeoutTracker; + + NettyClientHandler() { + timeoutTracker = Executors.newSingleThreadScheduledExecutor(new NamedThreadFactory("qmq-client-clean")); + timeoutTracker.scheduleAtFixedRate(new Runnable() { + @Override + public void run() { + NettyClientHandler.this.processTimeouts(); + } + }, 3 * CLEAN_RESPONSE_TABLE_PERIOD_MILLIS, CLEAN_RESPONSE_TABLE_PERIOD_MILLIS, TimeUnit.MILLISECONDS); + } + + ResponseFuture newResponse(Channel channel, long timeout, ResponseFuture.Callback callback) throws ClientSendException { + final int op = opaque.getAndIncrement(); + ResponseFuture future = new ResponseFuture(op, timeout, callback); + ConcurrentMap channelBuffer = requestsInFlight.get(channel); + if (channelBuffer == null) { + channelBuffer = new ConcurrentHashMap<>(); + ConcurrentMap old = requestsInFlight.putIfAbsent(channel, channelBuffer); + if (old != null) { + channelBuffer = old; + } + } + + if (channelBuffer.putIfAbsent(op, future) != null) { + throw new ClientSendException(ClientSendException.SendErrorCode.ILLEGAL_OPAQUE); + } + return future; + } + + void removeResponse(Channel channel, ResponseFuture responseFuture) { + ConcurrentMap channelBuffer = requestsInFlight.get(channel); + if (channelBuffer == null) return; + + channelBuffer.remove(responseFuture.getOpaque(), responseFuture); + } + + @Override + protected void channelRead0(ChannelHandlerContext ctx, Datagram datagram) { + if (datagram == null) return; + + try { + processResponse(ctx, datagram); + } catch (Exception e) { + LOGGER.error("processResponse exception", e); + } + } + + @Override + public void channelInactive(ChannelHandlerContext ctx) { + ConcurrentMap channelBuffer = requestsInFlight.remove(ctx.channel()); + if (channelBuffer == null) return; + + for (Map.Entry entry : channelBuffer.entrySet()) { + ResponseFuture responseFuture = entry.getValue(); + responseFuture.completeByTimeoutClean(); + responseFuture.executeCallbackOnlyOnce(); + } + } + + private void processResponse(ChannelHandlerContext ctx, Datagram response) { + int opaque = response.getHeader().getOpaque(); + ConcurrentMap channelBuffer = requestsInFlight.get(ctx.channel()); + if (channelBuffer == null) return; + + ResponseFuture responseFuture = channelBuffer.remove(opaque); + if (responseFuture != null) { + responseFuture.completeByReceiveResponse(response); + responseFuture.executeCallbackOnlyOnce(); + } else { + LOGGER.warn("receive response, but not matched any request, maybe response timeout or channel had been closed, {}", RemoteHelper.parseChannelRemoteAddress(ctx.channel())); + } + } + + private void processTimeouts() { + final List rfList = new LinkedList<>(); + Iterator>> channelBuffers = this.requestsInFlight.entrySet().iterator(); + while (channelBuffers.hasNext()) { + Map.Entry> channelBuffer = channelBuffers.next(); + Iterator> iterator = channelBuffer.getValue().entrySet().iterator(); + while (iterator.hasNext()) { + Map.Entry next = iterator.next(); + ResponseFuture future = next.getValue(); + + if (isTimeout(future)) { + future.completeByTimeoutClean(); + iterator.remove(); + + rfList.add(future); + LOGGER.warn("remove timeout request, " + future); + } + + } + } + + executeCallbacks(rfList); + } + + private boolean isTimeout(ResponseFuture future) { + return future.getTimeout() >= 0 && (future.getBeginTime() + future.getTimeout()) <= System.currentTimeMillis(); + } + + private void executeCallbacks(List rfList) { + for (ResponseFuture responseFuture : rfList) { + try { + responseFuture.executeCallbackOnlyOnce(); + } catch (Throwable e) { + LOGGER.warn("scanResponseTable, operationComplete Exception", e); + } + } + } + + void shutdown() { + timeoutTracker.shutdown(); + } +} diff --git a/qmq-remoting/src/main/java/qunar/tc/qmq/netty/client/NettyConnectManageHandler.java b/qmq-remoting/src/main/java/qunar/tc/qmq/netty/client/NettyConnectManageHandler.java new file mode 100644 index 00000000..18563044 --- /dev/null +++ b/qmq-remoting/src/main/java/qunar/tc/qmq/netty/client/NettyConnectManageHandler.java @@ -0,0 +1,214 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.netty.client; + +import com.google.common.base.Strings; +import com.google.common.util.concurrent.RateLimiter; +import io.netty.bootstrap.Bootstrap; +import io.netty.channel.*; +import io.netty.handler.timeout.IdleState; +import io.netty.handler.timeout.IdleStateEvent; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import qunar.tc.qmq.netty.exception.ClientSendException; +import qunar.tc.qmq.util.RemoteHelper; + +import java.net.SocketAddress; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.locks.Lock; +import java.util.concurrent.locks.ReentrantLock; + +/** + * @author yiqun.fan create on 17-8-29. + */ +@ChannelHandler.Sharable +public class NettyConnectManageHandler extends ChannelDuplexHandler { + private static final Logger LOGGER = LoggerFactory.getLogger(NettyConnectManageHandler.class); + private static final long LOCK_TIMEOUT_MILLIS = 5000; + + private final Bootstrap bootstrap; + private final long connectTimeout; + private final ConcurrentMap channelTables = new ConcurrentHashMap<>(); + private final Lock channelLock = new ReentrantLock(); + private final RateLimiter connectFailLogLimit = RateLimiter.create(0.2); + private final RateLimiter closeChannelLogLimit = RateLimiter.create(0.2); + + NettyConnectManageHandler(Bootstrap bootstrap, long connectTimeout) { + this.bootstrap = bootstrap; + this.connectTimeout = connectTimeout; + } + + void shutdown() { + for (ChannelWrapper cw : channelTables.values()) { + this.closeChannel(cw.getChannel()); + } + channelTables.clear(); + } + + private boolean tryLockChannelTable() { + try { + if (channelLock.tryLock(LOCK_TIMEOUT_MILLIS, TimeUnit.MILLISECONDS)) { + return true; + } else { + LOGGER.warn("try to lock channel table, but timeout in {}ms.", LOCK_TIMEOUT_MILLIS); + } + } catch (InterruptedException e) { + LOGGER.warn("try to lock channel table, but be interrupted."); + } + return false; + } + + Channel getOrCreateChannel(final String remoteAddr) throws ClientSendException { + if (Strings.isNullOrEmpty(remoteAddr)) { + throw new ClientSendException(ClientSendException.SendErrorCode.EMPTY_ADDRESS); + } + ChannelWrapper cw = channelTables.get(remoteAddr); + if (cw != null && cw.isOK()) { + return cw.getChannel(); + } + + if (!tryLockChannelTable()) { + throw new ClientSendException(ClientSendException.SendErrorCode.CREATE_CHANNEL_FAIL, remoteAddr); + } + try { + boolean needCreateChannel = true; + cw = channelTables.get(remoteAddr); + if (cw != null) { + if (cw.isOK()) { + return cw.getChannel(); + } else if (!cw.getChannelFuture().isDone()) { + needCreateChannel = false; + } else { + channelTables.remove(remoteAddr); + } + } + + if (needCreateChannel) { + ChannelFuture cf = bootstrap.connect(RemoteHelper.string2SocketAddress(remoteAddr)); + LOGGER.debug("NettyConnectManageHandler", "begin to connect remote host: {}", remoteAddr); + cw = new ChannelWrapper(cf); + channelTables.put(remoteAddr, cw); + } + } catch (Exception e) { + LOGGER.error("create channel exception. remoteAddr={}", remoteAddr, e); + } finally { + channelLock.unlock(); + } + + if (cw != null) { + ChannelFuture cf = cw.getChannelFuture(); + if (cf.awaitUninterruptibly(connectTimeout)) { + if (cw.isOK()) { + LOGGER.debug("NettyConnectManageHandler", "connect remote host success: {}", remoteAddr); + return cw.getChannel(); + } else { + if (connectFailLogLimit.tryAcquire()) { + LOGGER.warn("connect remote host fail: {}. {}", remoteAddr, cf.toString(), cf.cause()); + } + } + } else { + if (connectFailLogLimit.tryAcquire()) { + LOGGER.warn("connect remote host timeout: {}. {}", remoteAddr, cf.toString()); + } + } + } + throw new ClientSendException(ClientSendException.SendErrorCode.CREATE_CHANNEL_FAIL, remoteAddr); + } + + @Override + public void connect(ChannelHandlerContext ctx, SocketAddress remoteAddress, SocketAddress localAddress, ChannelPromise promise) throws Exception { + final String local = localAddress == null ? "UNKNOWN" : RemoteHelper.parseSocketAddressAddress(localAddress); + final String remote = remoteAddress == null ? "UNKNOWN" : RemoteHelper.parseSocketAddressAddress(remoteAddress); + LOGGER.debug("NettyConnectManageHandler", "NETTY CLIENT PIPELINE: CONNECT {} => {}", local, remote); + super.connect(ctx, remoteAddress, localAddress, promise); + } + + @Override + public void disconnect(ChannelHandlerContext ctx, ChannelPromise promise) throws Exception { + final String remoteAddress = RemoteHelper.parseChannelRemoteAddress(ctx.channel()); + LOGGER.debug("NettyConnectManageHandler", "NETTY CLIENT PIPELINE: DISCONNECT {}", remoteAddress); + closeChannel(ctx.channel()); + super.disconnect(ctx, promise); + } + + @Override + public void close(ChannelHandlerContext ctx, ChannelPromise promise) throws Exception { + final String remoteAddress = RemoteHelper.parseChannelRemoteAddress(ctx.channel()); + LOGGER.debug("NettyConnectManageHandler", "NETTY CLIENT PIPELINE: CLOSE {}", remoteAddress); + closeChannel(ctx.channel()); + super.close(ctx, promise); + } + + @Override + public void userEventTriggered(ChannelHandlerContext ctx, Object evt) throws Exception { + if (evt instanceof IdleStateEvent) { + IdleStateEvent event = (IdleStateEvent) evt; + if (event.state().equals(IdleState.ALL_IDLE)) { + final String remoteAddress = RemoteHelper.parseChannelRemoteAddress(ctx.channel()); + LOGGER.warn("NETTY CLIENT PIPELINE: IDLE exception [{}]", remoteAddress); + closeChannel(ctx.channel()); + } + } + } + + @Override + public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) throws Exception { + final String remoteAddress = RemoteHelper.parseChannelRemoteAddress(ctx.channel()); + LOGGER.warn("NETTY CLIENT PIPELINE: exceptionCaught {}", remoteAddress); + LOGGER.warn("NETTY CLIENT PIPELINE: exceptionCaught exception.", cause); + closeChannel(ctx.channel()); + } + + private void closeChannel(final Channel channel) { + if (channel == null) + return; + if (!tryLockChannelTable()) { + return; + } + try { + ChannelWrapper oldCw = null; + String remoteAddr = null; + for (Map.Entry entry : channelTables.entrySet()) { + ChannelWrapper cw = entry.getValue(); + if (cw != null && cw.getChannel() == channel) { + remoteAddr = entry.getKey(); + oldCw = cw; + break; + } + } + if (oldCw == null) { + LOGGER.debug("NettyConnectManageHandler", "close channel but not found in channelTable"); + return; + } + + channelTables.remove(remoteAddr); + if (closeChannelLogLimit.tryAcquire()) { + LOGGER.info("close channel and remove from channelTable. remoteAddr={}", remoteAddr); + RemoteHelper.closeChannel(channel, true); + } else { + RemoteHelper.closeChannel(channel, false); + } + } catch (Exception e) { + LOGGER.error("closeChannel: close the channel exception", e); + } finally { + this.channelLock.unlock(); + } + } +} diff --git a/qmq-remoting/src/main/java/qunar/tc/qmq/netty/client/Response.java b/qmq-remoting/src/main/java/qunar/tc/qmq/netty/client/Response.java new file mode 100644 index 00000000..2c29fbcc --- /dev/null +++ b/qmq-remoting/src/main/java/qunar/tc/qmq/netty/client/Response.java @@ -0,0 +1,28 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.netty.client; + +/** + * @author yiqun.fan create on 17-8-29. + */ +public interface Response { + int getStatusCode(); + + String getHeader(String name); + + String getBody(); +} diff --git a/qmq-remoting/src/main/java/qunar/tc/qmq/netty/client/ResponseFuture.java b/qmq-remoting/src/main/java/qunar/tc/qmq/netty/client/ResponseFuture.java new file mode 100644 index 00000000..c88aa3a5 --- /dev/null +++ b/qmq-remoting/src/main/java/qunar/tc/qmq/netty/client/ResponseFuture.java @@ -0,0 +1,126 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.netty.client; + +import qunar.tc.qmq.protocol.Datagram; + +import java.util.concurrent.atomic.AtomicBoolean; + +public class ResponseFuture { + private final int opaque; + private final long timeout; + private final Callback callback; + private final long beginTime = System.currentTimeMillis(); + private volatile long sentTime = -1; + private volatile long requestEndTime = -1; + + private volatile boolean sendOk = true; + private volatile boolean isTimeout = false; + private volatile Datagram response = null; + private volatile Throwable cause; + private final AtomicBoolean executeCallbackOnlyOnce = new AtomicBoolean(false); + + ResponseFuture(int opaque, long timeoutMs, Callback callback) { + this.opaque = opaque; + this.timeout = timeoutMs; + this.callback = callback; + } + + public int getOpaque() { + return opaque; + } + + public long getTimeout() { + return timeout; + } + + public long getBeginTime() { + return beginTime; + } + + public boolean isSendOk() { + return sendOk; + } + + void setSendOk(boolean sendOk) { + this.sentTime = System.currentTimeMillis(); + this.sendOk = sendOk; + } + + public long getRequestCostTime() { + return Math.max(0, requestEndTime - beginTime); + } + + public boolean isTimeout() { + return isTimeout; + } + + public Datagram getResponse() { + return response; + } + + public void completeBySendFail(Throwable cause) { + setSendOk(false); + this.requestEndTime = System.currentTimeMillis(); + this.cause = cause; + } + + public void completeByTimeoutClean() { + this.isTimeout = true; + this.requestEndTime = System.currentTimeMillis(); + } + + public void completeByReceiveResponse(final Datagram response) { + this.response = response; + this.requestEndTime = System.currentTimeMillis(); + } + + public Throwable getCause() { + return cause; + } + + public void setCause(Throwable cause) { + this.cause = cause; + } + + void executeCallbackOnlyOnce() { + if (callback == null) return; + + if (this.executeCallbackOnlyOnce.compareAndSet(false, true)) { + callback.processResponse(this); + } + } + + @Override + public String toString() { + return "ResponseFuture{" + + "opaque=" + opaque + ", " + + "begin=" + beginTime + ", " + + "send=" + sentTime + ", " + + "end=" + requestEndTime + ", " + + "response=" + response + ", " + + "sendOk=" + sendOk + ", " + + "isTimeout=" + isTimeout + ", " + + (isTimeout ? "timeout=" + timeout + ", " : "") + + "cause=" + cause + + "}"; + } + + public interface Callback { + void processResponse(ResponseFuture responseFuture); + } +} diff --git a/qmq-remoting/src/main/java/qunar/tc/qmq/netty/exception/BrokerRejectException.java b/qmq-remoting/src/main/java/qunar/tc/qmq/netty/exception/BrokerRejectException.java new file mode 100644 index 00000000..6d86c652 --- /dev/null +++ b/qmq-remoting/src/main/java/qunar/tc/qmq/netty/exception/BrokerRejectException.java @@ -0,0 +1,29 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.netty.exception; + +import qunar.tc.qmq.service.exceptions.MessageException; + +/** + * @author yiqun.fan create on 17-9-8. + */ +public class BrokerRejectException extends MessageException { + + public BrokerRejectException(String messageId) { + super(messageId, "reject"); + } +} diff --git a/qmq-remoting/src/main/java/qunar/tc/qmq/netty/exception/ClientSendException.java b/qmq-remoting/src/main/java/qunar/tc/qmq/netty/exception/ClientSendException.java new file mode 100644 index 00000000..8064427d --- /dev/null +++ b/qmq-remoting/src/main/java/qunar/tc/qmq/netty/exception/ClientSendException.java @@ -0,0 +1,63 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.netty.exception; + +/** + * @author yiqun.fan create on 17-7-5. + */ +public class ClientSendException extends Exception { + + private static final long serialVersionUID = 6006709158339785244L; + + private final SendErrorCode sendErrorCode; + private final String brokerAddr; + + public ClientSendException(SendErrorCode sendErrorCode) { + super(sendErrorCode.name()); + this.sendErrorCode = sendErrorCode; + this.brokerAddr = ""; + } + + public ClientSendException(SendErrorCode sendErrorCode, String brokerAddr) { + super(sendErrorCode.name() + ", broker address [" + brokerAddr + "]"); + this.sendErrorCode = sendErrorCode; + this.brokerAddr = brokerAddr; + } + + public ClientSendException(SendErrorCode sendErrorCode, String brokerAddr, Throwable cause) { + super(sendErrorCode.name() + ", broker address [" + brokerAddr + "]", cause); + this.sendErrorCode = sendErrorCode; + this.brokerAddr = brokerAddr; + } + + public SendErrorCode getSendErrorCode() { + return sendErrorCode; + } + + public String getBrokerAddr() { + return brokerAddr; + } + + public enum SendErrorCode { + ILLEGAL_OPAQUE, + EMPTY_ADDRESS, + CREATE_CHANNEL_FAIL, + CONNECT_BROKER_FAIL, + WRITE_CHANNEL_FAIL, + BROKER_BUSY + } +} diff --git a/qmq-remoting/src/main/java/qunar/tc/qmq/netty/exception/EncodeException.java b/qmq-remoting/src/main/java/qunar/tc/qmq/netty/exception/EncodeException.java new file mode 100644 index 00000000..227fbc05 --- /dev/null +++ b/qmq-remoting/src/main/java/qunar/tc/qmq/netty/exception/EncodeException.java @@ -0,0 +1,31 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.netty.exception; + +/** + * @author yiqun.fan create on 17-7-6. + */ +public class EncodeException extends Exception { + + public EncodeException(String s) { + super(s); + } + + public EncodeException(String s, Throwable cause) { + super(s, cause); + } +} diff --git a/qmq-remoting/src/main/java/qunar/tc/qmq/netty/exception/RemoteBusyException.java b/qmq-remoting/src/main/java/qunar/tc/qmq/netty/exception/RemoteBusyException.java new file mode 100644 index 00000000..a0584493 --- /dev/null +++ b/qmq-remoting/src/main/java/qunar/tc/qmq/netty/exception/RemoteBusyException.java @@ -0,0 +1,25 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.netty.exception; + +/** + * @author zhenyu.nie created on 2017 2017/7/6 18:16 + */ +public class RemoteBusyException extends RemoteException { + + private static final long serialVersionUID = -1246238157178008281L; +} diff --git a/qmq-remoting/src/main/java/qunar/tc/qmq/netty/exception/RemoteException.java b/qmq-remoting/src/main/java/qunar/tc/qmq/netty/exception/RemoteException.java new file mode 100644 index 00000000..c6907514 --- /dev/null +++ b/qmq-remoting/src/main/java/qunar/tc/qmq/netty/exception/RemoteException.java @@ -0,0 +1,40 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.netty.exception; + +/** + * @author zhenyu.nie created on 2017 2017/7/6 16:44 + */ +public class RemoteException extends Exception { + + private static final long serialVersionUID = -7144986221917657039L; + + public RemoteException() { + } + + public RemoteException(String message) { + super(message); + } + + public RemoteException(String message, Throwable cause) { + super(message, cause); + } + + public RemoteException(Throwable cause) { + super(cause); + } +} diff --git a/qmq-remoting/src/main/java/qunar/tc/qmq/netty/exception/RemoteResponseUnreadableException.java b/qmq-remoting/src/main/java/qunar/tc/qmq/netty/exception/RemoteResponseUnreadableException.java new file mode 100644 index 00000000..5e248ab4 --- /dev/null +++ b/qmq-remoting/src/main/java/qunar/tc/qmq/netty/exception/RemoteResponseUnreadableException.java @@ -0,0 +1,24 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.netty.exception; + +/** + * @author zhenyu.nie created on 2017 2017/7/10 16:11 + */ +public class RemoteResponseUnreadableException extends RemoteException { + private static final long serialVersionUID = -4706381924995618174L; +} diff --git a/qmq-remoting/src/main/java/qunar/tc/qmq/netty/exception/RemoteTimeoutException.java b/qmq-remoting/src/main/java/qunar/tc/qmq/netty/exception/RemoteTimeoutException.java new file mode 100644 index 00000000..55fae15b --- /dev/null +++ b/qmq-remoting/src/main/java/qunar/tc/qmq/netty/exception/RemoteTimeoutException.java @@ -0,0 +1,40 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.netty.exception; + +/** + * @author zhenyu.nie created on 2017 2017/7/6 16:44 + */ +public class RemoteTimeoutException extends RemoteException { + + private static final long serialVersionUID = -7427009787627990391L; + + public RemoteTimeoutException() { + } + + public RemoteTimeoutException(Throwable cause) { + super(cause); + } + + public RemoteTimeoutException(String address, long timeoutMs) { + this(address, timeoutMs, null); + } + + public RemoteTimeoutException(String address, long timeoutMs, Throwable cause) { + super("remote timeout on address [" + address + "] with timeout [" + timeoutMs + "]ms", cause); + } +} diff --git a/qmq-remoting/src/main/java/qunar/tc/qmq/netty/exception/SubjectNotAssignedException.java b/qmq-remoting/src/main/java/qunar/tc/qmq/netty/exception/SubjectNotAssignedException.java new file mode 100644 index 00000000..a7ad6bc9 --- /dev/null +++ b/qmq-remoting/src/main/java/qunar/tc/qmq/netty/exception/SubjectNotAssignedException.java @@ -0,0 +1,34 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.netty.exception; + +import qunar.tc.qmq.service.exceptions.MessageException; + +/** + * @author yiqun.fan create on 17-9-8. + */ +public class SubjectNotAssignedException extends MessageException { + + public SubjectNotAssignedException(String messageId) { + super(messageId, "subject not assigned"); + } + + @Override + public boolean isSubjectNotAssigned() { + return true; + } +} diff --git a/qmq-remoting/src/main/java/qunar/tc/qmq/protocol/CommandCode.java b/qmq-remoting/src/main/java/qunar/tc/qmq/protocol/CommandCode.java new file mode 100644 index 00000000..0cb22757 --- /dev/null +++ b/qmq-remoting/src/main/java/qunar/tc/qmq/protocol/CommandCode.java @@ -0,0 +1,59 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.protocol; + +/** + * @author yunfeng.yang + * @since 2017/6/30 + */ +public interface CommandCode { + short PLACEHOLDER = -1; + + // response code + short SUCCESS = 0; + short UNKNOWN_CODE = 3; + short NO_MESSAGE = 4; + + short BROKER_ERROR = 51; + short BROKER_REJECT = 52; + short PARAM_ERROR = 53; + + // request code + short SEND_MESSAGE = 10; + short PULL_MESSAGE = 11; + short ACK_REQUEST = 12; + + short SYNC_LOG_REQUEST = 20; + short SYNC_CHECKPOINT_REQUEST = 21; + + short QUERY_CONSUMER_LAG = 24; + short CONSUME_MANAGE = 25; + short QUEUE_COUNT = 26; + short ACTION_OFFSET_REQUEST = 27; + + short BROKER_REGISTER = 30; + short CLIENT_REGISTER = 35; + + short BROKER_ACQUIRE_META = 40; + + short UID_ASSIGN = 50; + short UID_ACQUIRE = 51; + + // heartbeat + short HEARTBEAT = 100; + +} diff --git a/qmq-remoting/src/main/java/qunar/tc/qmq/protocol/Datagram.java b/qmq-remoting/src/main/java/qunar/tc/qmq/protocol/Datagram.java new file mode 100644 index 00000000..79f6ddbf --- /dev/null +++ b/qmq-remoting/src/main/java/qunar/tc/qmq/protocol/Datagram.java @@ -0,0 +1,65 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.protocol; + +import io.netty.buffer.ByteBuf; +import io.netty.util.ReferenceCountUtil; + +/** + * @author yiqun.fan create on 17-7-4. + */ +public class Datagram { + RemotingHeader header; + private ByteBuf body; + private PayloadHolder holder; + + public ByteBuf getBody() { + return body; + } + + public void setBody(ByteBuf body) { + this.body = body; + } + + public void setPayloadHolder(PayloadHolder holder) { + this.holder = holder; + } + + public RemotingHeader getHeader() { + return header; + } + + public void setHeader(RemotingHeader header) { + this.header = header; + } + + public void writeBody(ByteBuf out) { + if (holder == null) return; + holder.writeBody(out); + } + + public void release() { + ReferenceCountUtil.safeRelease(body); + } + + @Override + public String toString() { + return "Datagram{" + + "header=" + header + + '}'; + } +} diff --git a/qmq-remoting/src/main/java/qunar/tc/qmq/protocol/MessagesPayloadHolder.java b/qmq-remoting/src/main/java/qunar/tc/qmq/protocol/MessagesPayloadHolder.java new file mode 100644 index 00000000..ca96b8bf --- /dev/null +++ b/qmq-remoting/src/main/java/qunar/tc/qmq/protocol/MessagesPayloadHolder.java @@ -0,0 +1,135 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.protocol; + +import io.netty.buffer.ByteBuf; +import qunar.tc.qmq.base.BaseMessage; +import qunar.tc.qmq.utils.Crc32; +import qunar.tc.qmq.utils.DelayUtil; +import qunar.tc.qmq.utils.Flags; +import qunar.tc.qmq.utils.PayloadHolderUtils; + +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +/** + * Created by zhaohui.yu + * 7/21/17 + */ +public class MessagesPayloadHolder implements PayloadHolder { + private final List messages; + + public MessagesPayloadHolder(List messages) { + this.messages = messages; + } + + @Override + public void writeBody(ByteBuf out) { + if (messages == null || messages.size() == 0) return; + for (BaseMessage message : messages) { + serializeMessage(message, out); + } + } + + private void serializeMessage(BaseMessage message, ByteBuf out) { + int crcIndex = out.writerIndex(); + // sizeof(bodyCrc) + out.ensureWritable(8); + out.writerIndex(crcIndex + 8); + + final int messageStart = out.writerIndex(); + + // flag + byte flag = 0; + //由低到高,第二位标识延迟(1)非延迟(0),第三位标识是(1)否(0)包含Tag + flag = Flags.setDelay(flag, DelayUtil.isDelayMessage(message)); + + //in avoid add tag after sendMessage + Set tags = new HashSet<>(message.getTags()); + flag = Flags.setTags(flag, hasTags(tags)); + + out.writeByte(flag); + + // created time + out.writeLong(message.getCreatedTime().getTime()); + if (Flags.isDelay(flag)) { + out.writeLong(message.getScheduleReceiveTime().getTime()); + } else { + // expired time + out.writeLong(System.currentTimeMillis()); + } + // subject + PayloadHolderUtils.writeString(message.getSubject(), out); + // message id + PayloadHolderUtils.writeString(message.getMessageId(), out); + + writeTags(tags, out); + + out.markWriterIndex(); + // writerIndex + sizeof(bodyLength) + final int bodyStart = out.writerIndex() + 4; + out.ensureWritable(4); + out.writerIndex(bodyStart); + + serializeMap(message.getAttrs(), out); + final int bodyEnd = out.writerIndex(); + + final int messageEnd = out.writerIndex(); + + final int bodyLen = bodyEnd - bodyStart; + final int messageLength = bodyEnd - messageStart; + + // write body length + out.resetWriterIndex(); + out.writeInt(bodyLen); + + // write message crc + out.writerIndex(crcIndex); + out.writeLong(messageCrc(out, messageStart, messageLength)); + + out.writerIndex(messageEnd); + } + + private void writeTags(Set tags, ByteBuf out) { + if (tags.isEmpty()) return; + out.writeByte((byte) tags.size()); + for (final String tag : tags) { + PayloadHolderUtils.writeString(tag, out); + } + } + + private boolean hasTags(Set tags) { + return tags.size() > 0; + } + + //TODO: 这里应该针对超大的消息记录监控 + private void serializeMap(Map map, ByteBuf out) { + if (null == map || map.isEmpty()) return; + + for (Map.Entry entry : map.entrySet()) { + if (entry.getKey() == null || entry.getValue() == null) continue; + PayloadHolderUtils.writeString(entry.getKey(), out); + PayloadHolderUtils.writeString(entry.getValue().toString(), out); + } + } + + private long messageCrc(ByteBuf out, int messageStart, int messageLength) { + return Crc32.crc32(out.nioBuffer(messageStart, messageLength), 0, messageLength); + } +} diff --git a/qmq-remoting/src/main/java/qunar/tc/qmq/protocol/PayloadHolder.java b/qmq-remoting/src/main/java/qunar/tc/qmq/protocol/PayloadHolder.java new file mode 100644 index 00000000..2a4c8454 --- /dev/null +++ b/qmq-remoting/src/main/java/qunar/tc/qmq/protocol/PayloadHolder.java @@ -0,0 +1,27 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.protocol; + +import io.netty.buffer.ByteBuf; + +/** + * Created by zhaohui.yu + * 7/21/17 + */ +public interface PayloadHolder { + void writeBody(ByteBuf out); +} diff --git a/qmq-remoting/src/main/java/qunar/tc/qmq/protocol/QMQSerializer.java b/qmq-remoting/src/main/java/qunar/tc/qmq/protocol/QMQSerializer.java new file mode 100644 index 00000000..ad24a916 --- /dev/null +++ b/qmq-remoting/src/main/java/qunar/tc/qmq/protocol/QMQSerializer.java @@ -0,0 +1,90 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.protocol; + +import com.google.common.collect.Maps; +import io.netty.buffer.ByteBuf; +import io.netty.buffer.Unpooled; +import qunar.tc.qmq.base.MessageHeader; +import qunar.tc.qmq.base.RawMessage; +import qunar.tc.qmq.protocol.producer.SendResult; +import qunar.tc.qmq.utils.Crc32; +import qunar.tc.qmq.utils.Flags; +import qunar.tc.qmq.utils.PayloadHolderUtils; + +import java.util.HashSet; +import java.util.Map; +import java.util.Set; + +/** + * @author yunfeng.yang + * @since 2017/7/14 + */ +public class QMQSerializer { + + public static RawMessage deserializeRawMessage(ByteBuf body) { + int headerStart = body.readerIndex(); + body.markReaderIndex(); + MessageHeader header = deserializeMessageHeader(body); + int bodyLen = body.readInt(); + int headerLen = body.readerIndex() - headerStart; + + int totalLen = headerLen + bodyLen; + body.resetReaderIndex(); + byte[] data = new byte[totalLen]; + body.readBytes(data); + header.setBodyCrc(Crc32.crc32(data)); + return new RawMessage(header, Unpooled.wrappedBuffer(data), totalLen); + } + + public static MessageHeader deserializeMessageHeader(ByteBuf body) { + byte flag = body.readByte(); + long createdTime = body.readLong(); + long expiredTime = body.readLong(); + String subject = PayloadHolderUtils.readString(body); + String messageId = PayloadHolderUtils.readString(body); + MessageHeader header = new MessageHeader(); + if (Flags.hasTags(flag)) { + final Set tags = new HashSet<>(); + final byte tagsSize = body.readByte(); + for (int i = 0; i < tagsSize; i++) { + String tag = PayloadHolderUtils.readString(body); + tags.add(tag); + } + header.setTags(tags); + } + + header.setFlag(flag); + header.setCreateTime(createdTime); + header.setExpireTime(expiredTime); + header.setSubject(subject); + header.setMessageId(messageId); + return header; + } + + public static Map deserializeSendResultMap(ByteBuf buf) { + Map result = Maps.newHashMap(); + while (buf.isReadable()) { + String messageId = PayloadHolderUtils.readString(buf); + int code = buf.readInt(); + String remark = PayloadHolderUtils.readString(buf); + result.put(messageId, new SendResult(code, remark)); + } + return result; + } + +} diff --git a/qmq-remoting/src/main/java/qunar/tc/qmq/protocol/RemotingCommand.java b/qmq-remoting/src/main/java/qunar/tc/qmq/protocol/RemotingCommand.java new file mode 100644 index 00000000..a36bf7b5 --- /dev/null +++ b/qmq-remoting/src/main/java/qunar/tc/qmq/protocol/RemotingCommand.java @@ -0,0 +1,48 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.protocol; + +/** + * @author yunfeng.yang + * @since 2017/7/3 + */ +public class RemotingCommand extends Datagram { + private long receiveTime; + + public RemotingCommandType getCommandType() { + int bits = 1; + int flag0 = this.header.getFlag() & bits; + return RemotingCommandType.codeOf(flag0); + } + + public boolean isOneWay() { + int bits = 1 << 1; + return (this.header.getFlag() & bits) == bits; + } + + public int getOpaque() { + return header.getOpaque(); + } + + public long getReceiveTime() { + return receiveTime; + } + + public void setReceiveTime(long receiveTime) { + this.receiveTime = receiveTime; + } +} diff --git a/qmq-remoting/src/main/java/qunar/tc/qmq/protocol/RemotingCommandType.java b/qmq-remoting/src/main/java/qunar/tc/qmq/protocol/RemotingCommandType.java new file mode 100644 index 00000000..4968e257 --- /dev/null +++ b/qmq-remoting/src/main/java/qunar/tc/qmq/protocol/RemotingCommandType.java @@ -0,0 +1,45 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.protocol; + +/** + * @author yunfeng.yang + * @since 2017/7/3 + */ +public enum RemotingCommandType { + REQUEST_COMMAND(0), + RESPONSE_COMMAND(1); + + private int code; + + RemotingCommandType(int flag) { + this.code = flag; + } + + public static RemotingCommandType codeOf(int code) { + for(RemotingCommandType domainType : RemotingCommandType.values()) { + if(domainType.code == code) { + return domainType; + } + } + throw new RuntimeException("Unsupported Command code"); + } + + public int getCode() { + return code; + } +} diff --git a/qmq-remoting/src/main/java/qunar/tc/qmq/protocol/RemotingHeader.java b/qmq-remoting/src/main/java/qunar/tc/qmq/protocol/RemotingHeader.java new file mode 100644 index 00000000..b59d0b88 --- /dev/null +++ b/qmq-remoting/src/main/java/qunar/tc/qmq/protocol/RemotingHeader.java @@ -0,0 +1,121 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.protocol; + +/** + * @author yiqun.fan create on 17-7-4. + */ +public class RemotingHeader { + public static final int DEFAULT_MAGIC_CODE = 0xdec1_0ade; + + public static final short VERSION_1 = 1; + public static final short VERSION_2 = 2; + public static final short VERSION_3 = 3; + public static final short VERSION_4 = 4; + /** + * (deprecated) compute crc only on message attr + */ + public static final short VERSION_5 = 5; + /** + * add crc while sending message to broker + */ + public static final short VERSION_6 = 6; + /** + * add schedule time in message header for delay message + */ + public static final short VERSION_7 = 7; + + /** + * add tags field for message header + */ + public static final short VERSION_8 = 8; + + public static final short MIN_HEADER_SIZE = 16; // magic code(4) + code(2) + version(2) + opaque(4) + flag(4) + public static final short HEADER_SIZE_LEN = 2; + public static final short TOTAL_SIZE_LEN = 4; + + public static final short LENGTH_FIELD = TOTAL_SIZE_LEN + HEADER_SIZE_LEN; + + public static final short REQUEST_CODE_LEN = 2; + + private int magicCode = DEFAULT_MAGIC_CODE; + private short code; + private short version = VERSION_8; + private int opaque; + private int flag; + private short requestCode = CommandCode.PLACEHOLDER; + + public int getMagicCode() { + return magicCode; + } + + public void setMagicCode(int magicCode) { + this.magicCode = magicCode; + } + + public short getCode() { + return code; + } + + public void setCode(short code) { + this.code = code; + } + + public short getVersion() { + return version; + } + + public void setVersion(short version) { + this.version = version; + } + + public int getOpaque() { + return opaque; + } + + public void setOpaque(int opaque) { + this.opaque = opaque; + } + + public int getFlag() { + return flag; + } + + public void setFlag(int flag) { + this.flag = flag; + } + + public short getRequestCode() { + return requestCode; + } + + public void setRequestCode(short requestCode) { + this.requestCode = requestCode; + } + + @Override + public String toString() { + return "RemotingHeader{" + + "magicCode=" + magicCode + + ", code=" + code + + ", version=" + version + + ", opaque=" + opaque + + ", flag=" + flag + + ", requestCode=" + requestCode + + '}'; + } +} diff --git a/qmq-remoting/src/main/java/qunar/tc/qmq/protocol/consumer/AckRequest.java b/qmq-remoting/src/main/java/qunar/tc/qmq/protocol/consumer/AckRequest.java new file mode 100644 index 00000000..0b118e28 --- /dev/null +++ b/qmq-remoting/src/main/java/qunar/tc/qmq/protocol/consumer/AckRequest.java @@ -0,0 +1,78 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.protocol.consumer; + +/** + * @author yiqun.fan create on 17-8-24. + */ +public class AckRequest { + private static final byte UNSET = -1; + private String subject; + private String group; + private String consumerId; + private long pullOffsetBegin; + private long pullOffsetLast; + private byte isBroadcast = UNSET; + + public String getSubject() { + return subject; + } + + public void setSubject(String subject) { + this.subject = subject; + } + + public String getGroup() { + return group; + } + + public void setGroup(String group) { + this.group = group; + } + + public String getConsumerId() { + return consumerId; + } + + public void setConsumerId(String consumerId) { + this.consumerId = consumerId; + } + + public long getPullOffsetBegin() { + return pullOffsetBegin; + } + + public void setPullOffsetBegin(long pullOffsetBegin) { + this.pullOffsetBegin = pullOffsetBegin; + } + + public long getPullOffsetLast() { + return pullOffsetLast; + } + + public void setPullOffsetLast(long pullOffsetLast) { + this.pullOffsetLast = pullOffsetLast; + } + + public void setBroadcast(byte isBroadcast) { + this.isBroadcast = isBroadcast; + } + + public byte isBroadcast() { + return isBroadcast; + } +} diff --git a/qmq-remoting/src/main/java/qunar/tc/qmq/protocol/consumer/AckRequestPayloadHolder.java b/qmq-remoting/src/main/java/qunar/tc/qmq/protocol/consumer/AckRequestPayloadHolder.java new file mode 100644 index 00000000..8e9ec461 --- /dev/null +++ b/qmq-remoting/src/main/java/qunar/tc/qmq/protocol/consumer/AckRequestPayloadHolder.java @@ -0,0 +1,42 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.protocol.consumer; + +import io.netty.buffer.ByteBuf; +import qunar.tc.qmq.protocol.PayloadHolder; +import qunar.tc.qmq.utils.PayloadHolderUtils; + +/** + * @author yiqun.fan create on 17-8-25. + */ +public class AckRequestPayloadHolder implements PayloadHolder { + private final AckRequest request; + + public AckRequestPayloadHolder(AckRequest request) { + this.request = request; + } + + @Override + public void writeBody(ByteBuf out) { + PayloadHolderUtils.writeString(request.getSubject(), out); + PayloadHolderUtils.writeString(request.getGroup(), out); + PayloadHolderUtils.writeString(request.getConsumerId(), out); + out.writeLong(request.getPullOffsetBegin()); + out.writeLong(request.getPullOffsetLast()); + out.writeByte(request.isBroadcast()); + } +} diff --git a/qmq-remoting/src/main/java/qunar/tc/qmq/protocol/consumer/MetaInfoRequest.java b/qmq-remoting/src/main/java/qunar/tc/qmq/protocol/consumer/MetaInfoRequest.java new file mode 100644 index 00000000..32f75bcf --- /dev/null +++ b/qmq-remoting/src/main/java/qunar/tc/qmq/protocol/consumer/MetaInfoRequest.java @@ -0,0 +1,138 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.protocol.consumer; + +import com.google.common.base.Strings; +import qunar.tc.qmq.base.ClientRequestType; +import qunar.tc.qmq.common.ClientType; + +import java.util.HashMap; +import java.util.Map; + +/** + * @author yiqun.fan create on 17-8-31. + */ +public class MetaInfoRequest { + private static final String SUBJECT = "subject"; + private static final String CLIENT_TYPE_CODE = "clientTypeCode"; + private static final String APP_CODE = "appCode"; + private static final String CLIENT_ID = "clientId"; + private static final String CONSUMER_GROUP = "consumerGroup"; + private static final String REQUEST_TYPE = "requestType"; + + private final Map attrs; + + public MetaInfoRequest() { + this.attrs = new HashMap<>(); + } + + public MetaInfoRequest(Map attrs) { + this.attrs = new HashMap<>(attrs); + } + + Map getAttrs() { + return attrs; + } + + public String getSubject() { + return Strings.nullToEmpty(attrs.get(SUBJECT)); + } + + public void setSubject(String subject) { + setStringValue(SUBJECT, subject); + } + + public int getClientTypeCode() { + return getIntValue(CLIENT_TYPE_CODE, ClientType.OTHER.getCode()); + } + + public void setClientType(ClientType clientType) { + setIntValue(CLIENT_TYPE_CODE, clientType.getCode()); + } + + public String getAppCode() { + return getStringValue(APP_CODE); + } + + public void setAppCode(String appCode) { + setStringValue(APP_CODE, appCode); + } + + public String getClientId() { + return Strings.nullToEmpty(attrs.get(CLIENT_ID)); + } + + public void setClientId(String clientId) { + setStringValue(CLIENT_ID, clientId); + } + + public String getConsumerGroup() { + return Strings.nullToEmpty(attrs.get(CONSUMER_GROUP)); + } + + public void setConsumerGroup(String consumerGroup) { + setStringValue(CONSUMER_GROUP, consumerGroup); + } + + public int getRequestType() { + return getIntValue(REQUEST_TYPE, ClientRequestType.ONLINE.getCode()); + } + + public void setRequestType(ClientRequestType requestType) { + setIntValue(REQUEST_TYPE, requestType.getCode()); + } + + private void setIntValue(String attrName, int value) { + attrs.put(attrName, Integer.toString(value)); + } + + private int getIntValue(String attrName, int defaultValue) { + try { + return Integer.parseInt(attrs.get(attrName)); + } catch (Exception e) { + return defaultValue; + } + } + + private void setStringValue(String attrName, String value) { + attrs.put(attrName, Strings.nullToEmpty(value)); + } + + private String getStringValue(String attrName) { + return Strings.nullToEmpty(attrs.get(attrName)); + } + + @Override + public String toString() { + return "MetaInfoRequest{" + "attrs='" + attrs + '}'; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (!(o instanceof MetaInfoRequest)) return false; + + MetaInfoRequest that = (MetaInfoRequest) o; + + return attrs != null ? attrs.equals(that.attrs) : that.attrs == null; + } + + @Override + public int hashCode() { + return attrs != null ? attrs.hashCode() : 0; + } +} diff --git a/qmq-remoting/src/main/java/qunar/tc/qmq/protocol/consumer/MetaInfoRequestPayloadHolder.java b/qmq-remoting/src/main/java/qunar/tc/qmq/protocol/consumer/MetaInfoRequestPayloadHolder.java new file mode 100644 index 00000000..7eae67d7 --- /dev/null +++ b/qmq-remoting/src/main/java/qunar/tc/qmq/protocol/consumer/MetaInfoRequestPayloadHolder.java @@ -0,0 +1,37 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.protocol.consumer; + +import io.netty.buffer.ByteBuf; +import qunar.tc.qmq.protocol.PayloadHolder; +import qunar.tc.qmq.utils.PayloadHolderUtils; + +/** + * @author yiqun.fan create on 17-8-31. + */ +public class MetaInfoRequestPayloadHolder implements PayloadHolder { + private final MetaInfoRequest request; + + public MetaInfoRequestPayloadHolder(MetaInfoRequest request) { + this.request = request; + } + + @Override + public void writeBody(ByteBuf out) { + PayloadHolderUtils.writeStringMap(request.getAttrs(), out); + } +} diff --git a/qmq-remoting/src/main/java/qunar/tc/qmq/protocol/consumer/MetaInfoResponse.java b/qmq-remoting/src/main/java/qunar/tc/qmq/protocol/consumer/MetaInfoResponse.java new file mode 100644 index 00000000..91972993 --- /dev/null +++ b/qmq-remoting/src/main/java/qunar/tc/qmq/protocol/consumer/MetaInfoResponse.java @@ -0,0 +1,92 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.protocol.consumer; + +import qunar.tc.qmq.meta.BrokerCluster; +import qunar.tc.qmq.base.OnOfflineState; + +/** + * @author yiqun.fan create on 17-8-31. + */ +public class MetaInfoResponse { + private long timestamp; + private String subject; + private String consumerGroup; + private OnOfflineState onOfflineState; + private int clientTypeCode; + private BrokerCluster brokerCluster; + + public long getTimestamp() { + return timestamp; + } + + public void setTimestamp(long timestamp) { + this.timestamp = timestamp; + } + + public String getSubject() { + return subject; + } + + public void setSubject(String subject) { + this.subject = subject; + } + + public String getConsumerGroup() { + return consumerGroup; + } + + public void setConsumerGroup(String consumerGroup) { + this.consumerGroup = consumerGroup; + } + + public OnOfflineState getOnOfflineState() { + return onOfflineState; + } + + public void setOnOfflineState(OnOfflineState onOfflineState) { + this.onOfflineState = onOfflineState; + } + + public int getClientTypeCode() { + return clientTypeCode; + } + + public void setClientTypeCode(int clientTypeCode) { + this.clientTypeCode = clientTypeCode; + } + + public BrokerCluster getBrokerCluster() { + return brokerCluster; + } + + public void setBrokerCluster(BrokerCluster brokerCluster) { + this.brokerCluster = brokerCluster; + } + + @Override + public String toString() { + return "MetaInfoResponse{" + + "timestamp=" + timestamp + + ", subject='" + subject + '\'' + + ", consumerGroup='" + consumerGroup + '\'' + + ", onOfflineState=" + onOfflineState + + ", clientTypeCode=" + clientTypeCode + + ", brokerCluster=" + brokerCluster + + '}'; + } +} diff --git a/qmq-remoting/src/main/java/qunar/tc/qmq/protocol/consumer/PullRequest.java b/qmq-remoting/src/main/java/qunar/tc/qmq/protocol/consumer/PullRequest.java new file mode 100644 index 00000000..9a5091a3 --- /dev/null +++ b/qmq-remoting/src/main/java/qunar/tc/qmq/protocol/consumer/PullRequest.java @@ -0,0 +1,147 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.protocol.consumer; + +import qunar.tc.qmq.TagType; + +import java.util.Collections; +import java.util.List; + +/** + * @author yiqun.fan create on 17-7-4. + */ +public class PullRequest { + private String subject; + private String group; + private int requestNum; + private long timeoutMillis; + private long offset; + private long pullOffsetBegin; + private long pullOffsetLast; + private String consumerId; + private boolean isBroadcast; + + private int tagTypeCode = TagType.NO_TAG.getCode(); + + private List tags = Collections.emptyList(); + + public String getSubject() { + return subject; + } + + public void setSubject(String subject) { + this.subject = subject; + } + + public String getGroup() { + return group; + } + + public void setGroup(String group) { + this.group = group; + } + + public int getRequestNum() { + return requestNum; + } + + public void setRequestNum(int requestNum) { + this.requestNum = requestNum; + } + + public long getTimeoutMillis() { + return timeoutMillis; + } + + public void setTimeoutMillis(long timeoutMillis) { + this.timeoutMillis = timeoutMillis; + } + + public long getOffset() { + return offset; + } + + public void setOffset(long offset) { + this.offset = offset; + } + + public long getPullOffsetBegin() { + return pullOffsetBegin; + } + + public void setPullOffsetBegin(long pullOffsetBegin) { + this.pullOffsetBegin = pullOffsetBegin; + } + + public long getPullOffsetLast() { + return pullOffsetLast; + } + + public void setPullOffsetLast(long pullOffsetLast) { + this.pullOffsetLast = pullOffsetLast; + } + + public void setConsumerId(String consumerId) { + this.consumerId = consumerId; + } + + public String getConsumerId() { + return consumerId; + } + + public boolean isBroadcast() { + return isBroadcast; + } + + public void setBroadcast(boolean broadcast) { + isBroadcast = broadcast; + } + + public int getTagTypeCode() { + return tagTypeCode; + } + + public void setTagTypeCode(int tagTypeCode) { + this.tagTypeCode = tagTypeCode; + } + + public List getTags() { + return tags; + } + + public void setTags(List tags) { + this.tags = tags; + } + + @Override + public String toString() { + return "PullRequest{" + + "subject='" + subject + '\'' + + ", group='" + group + '\'' + + ", requestNum=" + requestNum + + ", timeoutMillis=" + timeoutMillis + + ", offset=" + offset + + ", pullOffsetBegin=" + pullOffsetBegin + + ", pullOffsetLast=" + pullOffsetLast + + ", consumerId='" + consumerId + '\'' + + ", isBroadcast=" + isBroadcast + + ", tagTypeCode=" + tagTypeCode + + ", tags=" + tags + + '}'; + } + +} diff --git a/qmq-remoting/src/main/java/qunar/tc/qmq/protocol/consumer/PullRequestPayloadHolder.java b/qmq-remoting/src/main/java/qunar/tc/qmq/protocol/consumer/PullRequestPayloadHolder.java new file mode 100644 index 00000000..1445d8fe --- /dev/null +++ b/qmq-remoting/src/main/java/qunar/tc/qmq/protocol/consumer/PullRequestPayloadHolder.java @@ -0,0 +1,51 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.protocol.consumer; + +import io.netty.buffer.ByteBuf; +import qunar.tc.qmq.protocol.PayloadHolder; +import qunar.tc.qmq.utils.PayloadHolderUtils; + +/** + * @author yiqun.fan create on 17-8-2. + */ +public class PullRequestPayloadHolder implements PayloadHolder { + private final PullRequest request; + + public PullRequestPayloadHolder(PullRequest request) { + this.request = request; + } + + @Override + public void writeBody(ByteBuf out) { + PayloadHolderUtils.writeString(request.getSubject(), out); + PayloadHolderUtils.writeString(request.getGroup(), out); + PayloadHolderUtils.writeString(request.getConsumerId(), out); + out.writeInt(request.getRequestNum()); + out.writeLong(request.getOffset()); + out.writeLong(request.getPullOffsetBegin()); + out.writeLong(request.getPullOffsetLast()); + out.writeLong(request.getTimeoutMillis()); + out.writeByte(request.isBroadcast() ? 1 : 0); + out.writeShort(request.getTagTypeCode()); + out.writeByte(request.getTags().size()); + for (byte[] tag : request.getTags()) { + out.writeShort((short) tag.length); + out.writeBytes(tag); + } + } +} diff --git a/qmq-remoting/src/main/java/qunar/tc/qmq/protocol/producer/MessageProducerCode.java b/qmq-remoting/src/main/java/qunar/tc/qmq/protocol/producer/MessageProducerCode.java new file mode 100644 index 00000000..4aa81e5b --- /dev/null +++ b/qmq-remoting/src/main/java/qunar/tc/qmq/protocol/producer/MessageProducerCode.java @@ -0,0 +1,32 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.protocol.producer; + +/** + * @author zhenyu.nie created on 2017 2017/7/6 16:04 + */ +public class MessageProducerCode { + + public static final int SUCCESS = 0; + public static final int BROKER_BUSY = 1; + public static final int MESSAGE_DUPLICATE = 2; + public static final int SUBJECT_NOT_ASSIGNED = 3; + public static final int BROKER_READ_ONLY = 4; + public static final int BLOCK = 5; + public static final int STORE_ERROR = 6; + +} diff --git a/qmq-remoting/src/main/java/qunar/tc/qmq/protocol/producer/SendResult.java b/qmq-remoting/src/main/java/qunar/tc/qmq/protocol/producer/SendResult.java new file mode 100644 index 00000000..2b217b92 --- /dev/null +++ b/qmq-remoting/src/main/java/qunar/tc/qmq/protocol/producer/SendResult.java @@ -0,0 +1,50 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.protocol.producer; + +/** + * @author yunfeng.yang + * @since 2017/7/6 + */ +public class SendResult { + + public static final SendResult OK = new SendResult(MessageProducerCode.SUCCESS, ""); + + private final int code; + private final String remark; + + public SendResult(int code, String remark) { + this.code = code; + this.remark = remark; + } + + public int getCode() { + return code; + } + + public String getRemark() { + return remark; + } + + @Override + public String toString() { + return "SendResult{" + + "code=" + code + + ", remark='" + remark + '\'' + + '}'; + } +} diff --git a/qmq-remoting/src/main/java/qunar/tc/qmq/util/ChannelUtil.java b/qmq-remoting/src/main/java/qunar/tc/qmq/util/ChannelUtil.java new file mode 100644 index 00000000..50a12f79 --- /dev/null +++ b/qmq-remoting/src/main/java/qunar/tc/qmq/util/ChannelUtil.java @@ -0,0 +1,52 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.util; + +import io.netty.channel.Channel; +import io.netty.util.Attribute; +import io.netty.util.AttributeKey; + +/** + * @author yiqun.fan create on 17-8-29. + */ +public class ChannelUtil { + private static final AttributeKey DEFAULT_ATTRIBUTE = AttributeKey.valueOf("default"); + + public static Object getAttribute(Channel channel) { + synchronized (channel) { + Attribute attr = channel.attr(DEFAULT_ATTRIBUTE); + return attr != null ? attr.get() : null; + } + } + + public static boolean setAttributeIfAbsent(Channel channel, Object o) { + synchronized (channel) { + Attribute attr = channel.attr(DEFAULT_ATTRIBUTE); + if (attr == null || attr.get() == null) { + channel.attr(DEFAULT_ATTRIBUTE).set(o); + return true; + } + return false; + } + } + + public static void removeAttribute(Channel channel) { + synchronized (channel) { + channel.attr(DEFAULT_ATTRIBUTE).set(null); + } + } +} diff --git a/qmq-remoting/src/main/java/qunar/tc/qmq/util/RemoteHelper.java b/qmq-remoting/src/main/java/qunar/tc/qmq/util/RemoteHelper.java new file mode 100644 index 00000000..fbe79437 --- /dev/null +++ b/qmq-remoting/src/main/java/qunar/tc/qmq/util/RemoteHelper.java @@ -0,0 +1,76 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.util; + +import io.netty.channel.Channel; +import io.netty.channel.ChannelFuture; +import io.netty.channel.ChannelFutureListener; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.net.InetSocketAddress; +import java.net.SocketAddress; + +public class RemoteHelper { + private static final Logger LOG = LoggerFactory.getLogger(RemoteHelper.class); + + public static String parseChannelRemoteAddress(final Channel channel) { + if (channel == null) { + return ""; + } + + final SocketAddress remote = channel.remoteAddress(); + final String address = remote != null ? remote.toString() : ""; + + final int index = address.lastIndexOf("/"); + if (index < 0) { + return address; + } else { + return address.substring(index + 1); + } + } + + public static String parseSocketAddressAddress(SocketAddress address) { + if (address == null) { + return ""; + } + + final String addr = address.toString(); + if (addr.isEmpty()) { + return ""; + } else { + return addr.substring(1); + } + } + + public static SocketAddress string2SocketAddress(final String address) { + final String[] s = address.split(":"); + return new InetSocketAddress(s[0], Integer.parseInt(s[1])); + } + + public static void closeChannel(Channel channel, final boolean enableLog) { + final String remoteAddr = RemoteHelper.parseChannelRemoteAddress(channel); + channel.close().addListener(new ChannelFutureListener() { + @Override + public void operationComplete(ChannelFuture future) { + if (enableLog) { + LOG.info("close channel result: {}. isClosed={}", remoteAddr, future.isSuccess()); + } + } + }); + } +} diff --git a/qmq-remoting/src/main/java/qunar/tc/qmq/util/RemotingBuilder.java b/qmq-remoting/src/main/java/qunar/tc/qmq/util/RemotingBuilder.java new file mode 100644 index 00000000..c3f7f940 --- /dev/null +++ b/qmq-remoting/src/main/java/qunar/tc/qmq/util/RemotingBuilder.java @@ -0,0 +1,70 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.util; + +import qunar.tc.qmq.protocol.Datagram; +import qunar.tc.qmq.protocol.PayloadHolder; +import qunar.tc.qmq.protocol.RemotingCommandType; +import qunar.tc.qmq.protocol.RemotingHeader; + +/** + * @author yunfeng.yang + * @since 2017/7/6 + */ +public class RemotingBuilder { + public static Datagram buildRequestDatagram(short code, PayloadHolder payloadHolder) { + final Datagram datagram = new Datagram(); + datagram.setHeader(buildRemotingHeader(code, RemotingCommandType.REQUEST_COMMAND.getCode(), 0)); + datagram.setPayloadHolder(payloadHolder); + return datagram; + } + + private static RemotingHeader buildRemotingHeader(short code, int flag, int opaque) { + final RemotingHeader header = new RemotingHeader(); + header.setCode(code); + header.setFlag(flag); + header.setOpaque(opaque); + header.setRequestCode(code); + header.setVersion(RemotingHeader.VERSION_8); + return header; + } + + public static Datagram buildResponseDatagram(final short code, final RemotingHeader requestHeader, final PayloadHolder payloadHolder) { + final Datagram datagram = new Datagram(); + datagram.setHeader(buildResponseHeader(code, requestHeader)); + datagram.setPayloadHolder(payloadHolder); + return datagram; + } + + public static Datagram buildEmptyResponseDatagram(final short code, final RemotingHeader requestHeader) { + return buildResponseDatagram(code, requestHeader, null); + } + + public static RemotingHeader buildResponseHeader(final short code, final RemotingHeader requestHeader) { + return buildRemotingHeader(code, RemotingCommandType.RESPONSE_COMMAND.getCode(), requestHeader); + } + + private static RemotingHeader buildRemotingHeader(final short code, final int flag, final RemotingHeader requestHeader) { + final RemotingHeader header = new RemotingHeader(); + header.setCode(code); + header.setFlag(flag); + header.setOpaque(requestHeader.getOpaque()); + header.setVersion(requestHeader.getVersion()); + header.setRequestCode(requestHeader.getCode()); + return header; + } +} diff --git a/qmq-server-common/pom.xml b/qmq-server-common/pom.xml new file mode 100644 index 00000000..48fe0982 --- /dev/null +++ b/qmq-server-common/pom.xml @@ -0,0 +1,50 @@ + + + 4.0.0 + + + qmq + qunar.tc + 4.0.30 + + + qmq-server-common + + + + ${project.groupId} + qmq-common + + + ${project.groupId} + qmq-remoting + + + org.slf4j + slf4j-api + + + org.slf4j + jcl-over-slf4j + + + org.slf4j + log4j-over-slf4j + + + ch.qos.logback + logback-classic + + + ch.qos.logback + logback-core + + + + com.google.guava + guava + + + \ No newline at end of file diff --git a/qmq-server-common/src/main/java/qunar/tc/qmq/base/ActionLogOffsetRequest.java b/qmq-server-common/src/main/java/qunar/tc/qmq/base/ActionLogOffsetRequest.java new file mode 100644 index 00000000..964ba471 --- /dev/null +++ b/qmq-server-common/src/main/java/qunar/tc/qmq/base/ActionLogOffsetRequest.java @@ -0,0 +1,52 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.base; + +/** + * @author: leix.xie + * @date: 2018/9/14 16:06 + * @describe: + */ +public class ActionLogOffsetRequest { + private String subject; + private String group; + private String consumerId; + + public String getSubject() { + return subject; + } + + public void setSubject(String subject) { + this.subject = subject; + } + + public String getGroup() { + return group; + } + + public void setGroup(String group) { + this.group = group; + } + + public String getConsumerId() { + return consumerId; + } + + public void setConsumerId(String consumerId) { + this.consumerId = consumerId; + } +} diff --git a/qmq-server-common/src/main/java/qunar/tc/qmq/base/ActionLogOffsetResponse.java b/qmq-server-common/src/main/java/qunar/tc/qmq/base/ActionLogOffsetResponse.java new file mode 100644 index 00000000..3961278e --- /dev/null +++ b/qmq-server-common/src/main/java/qunar/tc/qmq/base/ActionLogOffsetResponse.java @@ -0,0 +1,122 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.base; + +import java.util.Objects; +import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.locks.Lock; +import java.util.concurrent.locks.ReentrantLock; + +/** + * @author yunfeng.yang + * @since 2017/8/1 + */ +public class ActionLogOffsetResponse { + private AtomicLong pullLogOffset; + private AtomicLong ackLogOffset; + private String brokerAddress; + + private Lock pullLock = new ReentrantLock(); + private Lock ackLock = new ReentrantLock(); + + public ActionLogOffsetResponse(final long pullLogOffset, final long ackLogOffset) { + this.pullLogOffset = new AtomicLong(pullLogOffset); + this.ackLogOffset = new AtomicLong(ackLogOffset); + } + + public long getPullLogOffset() { + return pullLogOffset.get(); + } + + public void setPullLogOffset(long pullLogOffset) { + this.pullLogOffset.set(pullLogOffset); + } + + public long getAckLogOffset() { + return ackLogOffset.get(); + } + + public void setAckLogOffset(long ackLogOffset) { + this.ackLogOffset.set(ackLogOffset); + } + + public void pullLock() { + pullLock.lock(); + } + + public boolean tryPullLock() { + return pullLock.tryLock(); + } + + public void pullUnlock() { + pullLock.unlock(); + } + + public void ackLock() { + ackLock.lock(); + } + + public boolean tryAckLock() { + return ackLock.tryLock(); + } + + public void ackUnLock() { + ackLock.unlock(); + } + + public String getBrokerAddress() { + return brokerAddress; + } + + public void setBrokerAddress(String brokerAddress) { + this.brokerAddress = brokerAddress; + } + + @Override + public String toString() { + return "ActionLogOffsetResponse{" + + "pullLogOffset=" + pullLogOffset + + ", ackLogOffset=" + ackLogOffset + + ", brokerAddress='" + brokerAddress + '\'' + + '}'; + } + + @Override + public int hashCode() { + + return Objects.hash(pullLogOffset, ackLogOffset, brokerAddress); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) { + return false; + } + + ActionLogOffsetResponse that = (ActionLogOffsetResponse) o; + + if (!pullLogOffset.equals(that.pullLogOffset)) { + return false; + } + if (!ackLogOffset.equals(that.ackLogOffset)) { + return false; + } + return brokerAddress.equals(that.brokerAddress); + } + +} diff --git a/qmq-server-common/src/main/java/qunar/tc/qmq/base/ConsumeManageRequest.java b/qmq-server-common/src/main/java/qunar/tc/qmq/base/ConsumeManageRequest.java new file mode 100644 index 00000000..ccafedcc --- /dev/null +++ b/qmq-server-common/src/main/java/qunar/tc/qmq/base/ConsumeManageRequest.java @@ -0,0 +1,60 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.base; + +/** + * @author yunfeng.yang + * @since 2017/11/22 + */ +public class ConsumeManageRequest { + private int consumerFromWhere; + private String subject; + private String group; + + public int getConsumerFromWhere() { + return consumerFromWhere; + } + + public void setConsumerFromWhere(int consumerFromWhere) { + this.consumerFromWhere = consumerFromWhere; + } + + public String getSubject() { + return subject; + } + + public void setSubject(String subject) { + this.subject = subject; + } + + public String getGroup() { + return group; + } + + public void setGroup(String group) { + this.group = group; + } + + @Override + public String toString() { + return "ConsumeManageRequest{" + + "consumerFromWhere=" + consumerFromWhere + + ", subject='" + subject + '\'' + + ", group='" + group + '\'' + + '}'; + } +} diff --git a/qmq-server-common/src/main/java/qunar/tc/qmq/base/ConsumerLag.java b/qmq-server-common/src/main/java/qunar/tc/qmq/base/ConsumerLag.java new file mode 100644 index 00000000..d1403209 --- /dev/null +++ b/qmq-server-common/src/main/java/qunar/tc/qmq/base/ConsumerLag.java @@ -0,0 +1,43 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.base; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; + +/** + * @author keli.wang + * @since 2018/7/31 + */ +public class ConsumerLag { + private final long pull; + private final long ack; + + @JsonCreator + public ConsumerLag(@JsonProperty("pull") final long pull, @JsonProperty("ack") final long ack) { + this.pull = pull; + this.ack = ack; + } + + public long getPull() { + return pull; + } + + public long getAck() { + return ack; + } +} diff --git a/qmq-server-common/src/main/java/qunar/tc/qmq/base/QueryConsumerLagRequest.java b/qmq-server-common/src/main/java/qunar/tc/qmq/base/QueryConsumerLagRequest.java new file mode 100644 index 00000000..4b418bcb --- /dev/null +++ b/qmq-server-common/src/main/java/qunar/tc/qmq/base/QueryConsumerLagRequest.java @@ -0,0 +1,42 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.base; + +/** + * @author keli.wang + * @since 2018/7/31 + */ +public class QueryConsumerLagRequest { + private String subject; + private String group; + + public String getSubject() { + return subject; + } + + public void setSubject(String subject) { + this.subject = subject; + } + + public String getGroup() { + return group; + } + + public void setGroup(String group) { + this.group = group; + } +} diff --git a/qmq-server-common/src/main/java/qunar/tc/qmq/base/SyncRequest.java b/qmq-server-common/src/main/java/qunar/tc/qmq/base/SyncRequest.java new file mode 100644 index 00000000..1f52154e --- /dev/null +++ b/qmq-server-common/src/main/java/qunar/tc/qmq/base/SyncRequest.java @@ -0,0 +1,54 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.base; + +/** + * @author yunfeng.yang + * @since 2017/8/18 + */ +public class SyncRequest { + private final long messageLogOffset; + private final long actionLogOffset; + private final int syncType; + + public SyncRequest(int syncType, long messageLogOffset, long actionLogOffset) { + this.syncType = syncType; + this.messageLogOffset = messageLogOffset; + this.actionLogOffset = actionLogOffset; + } + + public int getSyncType() { + return syncType; + } + + public long getMessageLogOffset() { + return messageLogOffset; + } + + public long getActionLogOffset() { + return actionLogOffset; + } + + @Override + public String toString() { + return "SyncRequest{" + + "messageLogOffset=" + messageLogOffset + + ", actionLogOffset=" + actionLogOffset + + ", syncType=" + syncType + + '}'; + } +} diff --git a/qmq-server-common/src/main/java/qunar/tc/qmq/concurrent/ActorSystem.java b/qmq-server-common/src/main/java/qunar/tc/qmq/concurrent/ActorSystem.java new file mode 100644 index 00000000..1431d3b8 --- /dev/null +++ b/qmq-server-common/src/main/java/qunar/tc/qmq/concurrent/ActorSystem.java @@ -0,0 +1,452 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.concurrent; + +import com.google.common.collect.Maps; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import qunar.tc.qmq.monitor.QMon; + +import java.lang.reflect.Field; +import java.util.Objects; +import java.util.concurrent.*; +import java.util.concurrent.atomic.AtomicInteger; + +/** + * Created by zhaohui.yu + * 16/4/8 + */ +public class ActorSystem { + private static final Logger LOG = LoggerFactory.getLogger(ActorSystem.class); + + private static final int DEFAULT_QUEUE_SIZE = 10000; + + private final ConcurrentMap actors; + private final ThreadPoolExecutor executor; + private final AtomicInteger actorsCount; + private final String name; + + public ActorSystem(String name) { + this(name, Runtime.getRuntime().availableProcessors() * 4, true); + } + + public ActorSystem(String name, int threads, boolean fair) { + this.name = name; + this.actorsCount = new AtomicInteger(); + BlockingQueue queue = fair ? new PriorityBlockingQueue<>() : new LinkedBlockingQueue<>(); + this.executor = new ThreadPoolExecutor(threads, threads, 60, TimeUnit.MINUTES, queue, new NamedThreadFactory("actor-sys-" + name)); + this.actors = Maps.newConcurrentMap(); + QMon.dispatchersGauge(name, actorsCount::doubleValue); + QMon.actorSystemQueueGauge(name, () -> (double) executor.getQueue().size()); + } + + public void dispatch(String actorPath, E msg, Processor processor) { + Actor actor = createOrGet(actorPath, processor); + actor.dispatch(msg); + schedule(actor, true); + } + + public void suspend(String actorPath) { + Actor actor = actors.get(actorPath); + if (actor == null) return; + + actor.suspend(); + } + + public void resume(String actorPath) { + Actor actor = actors.get(actorPath); + if (actor == null) return; + + actor.resume(); + schedule(actor, false); + } + + private Actor createOrGet(String actorPath, Processor processor) { + Actor actor = actors.get(actorPath); + if (actor != null) return actor; + + Actor add = new Actor<>(this.name, actorPath, this, processor, DEFAULT_QUEUE_SIZE); + Actor old = actors.putIfAbsent(actorPath, add); + if (old == null) { + LOG.info("create actorSystem: {}", actorPath); + actorsCount.incrementAndGet(); + return add; + } + return old; + } + + private boolean schedule(Actor actor, boolean hasMessageHint) { + if (!actor.canBeSchedule(hasMessageHint)) return false; + if (actor.setAsScheduled()) { + actor.submitTs = System.currentTimeMillis(); + this.executor.execute(actor); + return true; + } + return false; + } + + public interface Processor { + boolean process(T message, Actor self); + } + + public static class Actor implements Runnable, Comparable { + private static final int Open = 0; + private static final int Scheduled = 2; + private static final int shouldScheduleMask = 3; + private static final int shouldNotProcessMask = ~2; + private static final int suspendUnit = 4; + //每个actor至少执行的时间片 + private static final int QUOTA = 5; + private static long statusOffset; + + static { + try { + statusOffset = Unsafe.instance.objectFieldOffset(Actor.class.getDeclaredField("status")); + } catch (Throwable t) { + throw new ExceptionInInitializerError(t); + } + } + + final String systemName; + final ActorSystem actorSystem; + final BoundedNodeQueue queue; + final Processor processor; + private final String name; + private long total; + private volatile long submitTs; + //通过Unsafe操作 + private volatile int status; + + Actor(String systemName, String name, ActorSystem actorSystem, Processor processor, final int queueSize) { + this.systemName = systemName; + this.name = name; + this.actorSystem = actorSystem; + this.processor = processor; + this.queue = new BoundedNodeQueue<>(queueSize); + + QMon.actorQueueGauge(systemName, name, () -> (double) queue.count()); + } + + boolean dispatch(E message) { + return queue.add(message); + } + + @Override + public void run() { + long start = System.currentTimeMillis(); + String old = Thread.currentThread().getName(); + try { + Thread.currentThread().setName(systemName + "-" + name); + if (shouldProcessMessage()) { + processMessages(); + } + } finally { + long duration = System.currentTimeMillis() - start; + total += duration; + QMon.actorProcessTime(name, duration); + + Thread.currentThread().setName(old); + setAsIdle(); + this.actorSystem.schedule(this, false); + } + } + + void processMessages() { + long deadline = System.currentTimeMillis() + QUOTA; + while (true) { + E message = queue.peek(); + if (message == null) return; + boolean process = processor.process(message, this); + if (!process) return; + + queue.pollNode(); + if (System.currentTimeMillis() >= deadline) return; + } + } + + final boolean shouldProcessMessage() { + return (currentStatus() & shouldNotProcessMask) == 0; + } + + private boolean canBeSchedule(boolean hasMessageHint) { + int s = currentStatus(); + if (s == Open || s == Scheduled) return hasMessageHint || !queue.isEmpty(); + return false; + } + + public final boolean resume() { + while (true) { + int s = currentStatus(); + int next = s < suspendUnit ? s : s - suspendUnit; + if (updateStatus(s, next)) return next < suspendUnit; + } + } + + public final void suspend() { + while (true) { + int s = currentStatus(); + if (updateStatus(s, s + suspendUnit)) return; + } + } + + final boolean setAsScheduled() { + while (true) { + int s = currentStatus(); + if ((s & shouldScheduleMask) != Open) return false; + if (updateStatus(s, s | Scheduled)) return true; + } + } + + final void setAsIdle() { + while (true) { + int s = currentStatus(); + if (updateStatus(s, s & ~Scheduled)) return; + } + } + + final int currentStatus() { + return Unsafe.instance.getIntVolatile(this, statusOffset); + } + + private boolean updateStatus(int oldStatus, int newStatus) { + return Unsafe.instance.compareAndSwapInt(this, statusOffset, oldStatus, newStatus); + } + + @Override + public int compareTo(Actor o) { + int result = Long.compare(total, o.total); + return result == 0 ? Long.compare(submitTs, o.submitTs) : result; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + Actor actor = (Actor) o; + return Objects.equals(systemName, actor.systemName) && + Objects.equals(name, actor.name); + } + + @Override + public int hashCode() { + return Objects.hash(systemName, name); + } + } + + /** + * Copyright (C) 2009-2016 Lightbend Inc. + */ + + /** + * Lock-free bounded non-blocking multiple-producer single-consumer queue based on the works of: + *

+ * Andriy Plokhotnuyk (https://github.com/plokhotnyuk) + * - https://github.com/plokhotnyuk/actors/blob/2e65abb7ce4cbfcb1b29c98ee99303d6ced6b01f/src/test/scala/akka/dispatch/Mailboxes.scala + * (Apache V2: https://github.com/plokhotnyuk/actors/blob/master/LICENSE) + *

+ * Dmitriy Vyukov's non-intrusive MPSC queue: + * - http://www.1024cores.net/home/lock-free-algorithms/queues/non-intrusive-mpsc-node-based-queue + * (Simplified BSD) + */ + @SuppressWarnings("serial") + private static class BoundedNodeQueue { + + private final static long enqOffset, deqOffset; + + static { + try { + enqOffset = Unsafe.instance.objectFieldOffset(BoundedNodeQueue.class.getDeclaredField("_enqDoNotCallMeDirectly")); + deqOffset = Unsafe.instance.objectFieldOffset(BoundedNodeQueue.class.getDeclaredField("_deqDoNotCallMeDirectly")); + } catch (Throwable t) { + throw new ExceptionInInitializerError(t); + } + } + + private final int capacity; + @SuppressWarnings("unused") + private volatile Node _enqDoNotCallMeDirectly; + @SuppressWarnings("unused") + private volatile Node _deqDoNotCallMeDirectly; + + protected BoundedNodeQueue(final int capacity) { + if (capacity < 0) throw new IllegalArgumentException("AbstractBoundedNodeQueue.capacity must be >= 0"); + this.capacity = capacity; + final Node n = new Node(); + setDeq(n); + setEnq(n); + } + + @SuppressWarnings("unchecked") + private Node getEnq() { + return (Node) Unsafe.instance.getObjectVolatile(this, enqOffset); + } + + private void setEnq(Node n) { + Unsafe.instance.putObjectVolatile(this, enqOffset, n); + } + + private boolean casEnq(Node old, Node nju) { + return Unsafe.instance.compareAndSwapObject(this, enqOffset, old, nju); + } + + @SuppressWarnings("unchecked") + private Node getDeq() { + return (Node) Unsafe.instance.getObjectVolatile(this, deqOffset); + } + + private void setDeq(Node n) { + Unsafe.instance.putObjectVolatile(this, deqOffset, n); + } + + private boolean casDeq(Node old, Node nju) { + return Unsafe.instance.compareAndSwapObject(this, deqOffset, old, nju); + } + + public final int count() { + final Node lastNode = getEnq(); + final int lastNodeCount = lastNode.count; + return lastNodeCount - getDeq().count; + } + + /** + * @return the maximum capacity of this queue + */ + public final int capacity() { + return capacity; + } + + // Possible TODO — impl. could be switched to addNode(new Node(value)) if we want to allocate even if full already + public final boolean add(final T value) { + for (Node n = null; ; ) { + final Node lastNode = getEnq(); + final int lastNodeCount = lastNode.count; + if (lastNodeCount - getDeq().count < capacity) { + // Trade a branch for avoiding to create a new node if full, + // and to avoid creating multiple nodes on write conflict á la Be Kind to Your GC + if (n == null) { + n = new Node(); + n.value = value; + } + + n.count = lastNodeCount + 1; // Piggyback on the HB-edge between getEnq() and casEnq() + + // Try to putPullLogs the node to the end, if we fail we continue loopin' + if (casEnq(lastNode, n)) { + lastNode.setNext(n); + return true; + } + } else return false; // Over capacity—couldn't add the node + } + } + + public final boolean isEmpty() { + return getEnq() == getDeq(); + } + + /** + * Removes the first element of this queue if any + * + * @return the value of the first element of the queue, null if empty + */ + public final T poll() { + final Node n = pollNode(); + return (n != null) ? n.value : null; + } + + public final T peek() { + Node n = peekNode(); + return (n != null) ? n.value : null; + } + + @SuppressWarnings("unchecked") + protected final Node peekNode() { + for (; ; ) { + final Node deq = getDeq(); + final Node next = deq.next(); + if (next != null || getEnq() == deq) + return next; + } + } + + /** + * Removes the first element of this queue if any + * + * @return the `Node` of the first element of the queue, null if empty + */ + public final Node pollNode() { + for (; ; ) { + final Node deq = getDeq(); + final Node next = deq.next(); + if (next != null) { + if (casDeq(deq, next)) { + deq.value = next.value; + deq.setNext(null); + next.value = null; + return deq; + } // else we retry (concurrent consumers) + } else if (getEnq() == deq) return null; // If we got a null and head meets tail, we are empty + } + } + + public static class Node { + private final static long nextOffset; + + static { + try { + nextOffset = Unsafe.instance.objectFieldOffset(Node.class.getDeclaredField("_nextDoNotCallMeDirectly")); + } catch (Throwable t) { + throw new ExceptionInInitializerError(t); + } + } + + protected T value; + protected int count; + @SuppressWarnings("unused") + private volatile Node _nextDoNotCallMeDirectly; + + @SuppressWarnings("unchecked") + public final Node next() { + return (Node) Unsafe.instance.getObjectVolatile(this, nextOffset); + } + + protected final void setNext(final Node newNext) { + Unsafe.instance.putOrderedObject(this, nextOffset, newNext); + } + } + } + + static class Unsafe { + public final static sun.misc.Unsafe instance; + + static { + try { + sun.misc.Unsafe found = null; + for (Field field : sun.misc.Unsafe.class.getDeclaredFields()) { + if (field.getType() == sun.misc.Unsafe.class) { + field.setAccessible(true); + found = (sun.misc.Unsafe) field.get(null); + break; + } + } + if (found == null) throw new IllegalStateException("Can't find instance of sun.misc.Unsafe"); + else instance = found; + } catch (Throwable t) { + throw new ExceptionInInitializerError(t); + } + } + } +} diff --git a/qmq-server-common/src/main/java/qunar/tc/qmq/configuration/BrokerConfig.java b/qmq-server-common/src/main/java/qunar/tc/qmq/configuration/BrokerConfig.java new file mode 100644 index 00000000..b0c4189e --- /dev/null +++ b/qmq-server-common/src/main/java/qunar/tc/qmq/configuration/BrokerConfig.java @@ -0,0 +1,86 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.configuration; + +import com.google.common.eventbus.Subscribe; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import qunar.tc.qmq.meta.BrokerAcquireMetaResponse; +import qunar.tc.qmq.meta.BrokerRole; +import qunar.tc.qmq.utils.NetworkUtils; + +/** + * @author yunfeng.yang + * @since 2017/8/19 + */ +public final class BrokerConfig { + private static final Logger LOG = LoggerFactory.getLogger(BrokerConfig.class); + + private static final BrokerConfig CONFIG = new BrokerConfig(); + + private volatile String brokerName; + private volatile BrokerRole brokerRole; + private volatile String brokerAddress; + private volatile String masterAddress; + private volatile boolean readonly; + + private BrokerConfig() { + brokerName = ""; + brokerRole = BrokerRole.STANDBY; + brokerAddress = NetworkUtils.getLocalAddress(); + masterAddress = ""; + readonly = true; + } + + public static BrokerConfig getInstance() { + return CONFIG; + } + + public static String getBrokerName() { + return CONFIG.brokerName; + } + + public static BrokerRole getBrokerRole() { + return CONFIG.brokerRole; + } + + public static String getBrokerAddress() { + return CONFIG.brokerAddress; + } + + public static String getMasterAddress() { + return CONFIG.masterAddress; + } + + public static boolean isReadonly() { + return CONFIG.readonly; + } + + public static void markAsWritable() { + CONFIG.readonly = false; + } + + @Subscribe + public void updateMeta(final BrokerAcquireMetaResponse response) { + LOG.info("Broker meta updated. meta: {}", response); + if (response.getRole() != BrokerRole.STANDBY) { + brokerRole = response.getRole(); + brokerName = response.getName(); + masterAddress = response.getMaster(); + } + } +} diff --git a/qmq-server-common/src/main/java/qunar/tc/qmq/constants/BrokerConstants.java b/qmq-server-common/src/main/java/qunar/tc/qmq/constants/BrokerConstants.java new file mode 100644 index 00000000..7346e639 --- /dev/null +++ b/qmq-server-common/src/main/java/qunar/tc/qmq/constants/BrokerConstants.java @@ -0,0 +1,57 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.constants; + +/** + * User: zhaohuiyu Date: 5/13/13 Time: 4:05 PM + */ +public class BrokerConstants { + public static final String PORT_CONFIG = "broker.port"; + public static final Integer DEFAULT_PORT = 20881; + + public static final String META_SERVER_ENDPOINT = "meta.server.endpoint"; + + public static final String STORE_ROOT = "store.root"; + public static final String LOG_STORE_ROOT = "/data"; + + public static final String MESSAGE_LOG_RETENTION_HOURS = "messagelog.retention.hours"; + public static final int DEFAULT_MESSAGE_LOG_RETENTION_HOURS = 72; // 3 days + + public static final String CONSUMER_LOG_RETENTION_HOURS = "consumerlog.retention.hours"; + public static final int DEFAULT_CONSUMER_LOG_RETENTION_HOURS = 72; // 3 days + + public static final String RETRY_DELAY_SECONDS = "message.retry.delay.seconds"; + public static final int DEFAULT_RETRY_DELAY_SECONDS = 5; + public static final String LOG_RETENTION_CHECK_INTERVAL_SECONDS = "log.retention.check.interval.seconds"; + public static final int DEFAULT_LOG_RETENTION_CHECK_INTERVAL_SECONDS = 60; + public static final String ENABLE_DELETE_EXPIRED_LOGS = "log.expired.delete.enable"; + + // slave + public static final long DEFAULT_HEARTBEAT_SLEEP_TIMEOUT_MS = 1000L; + + public static String PULL_LOG_RETENTION_HOURS = "pulllog.retention.hours"; + public static int DEFAULT_PULL_LOG_RETENTION_HOURS = 72; // 3 days + + public static String CHECKPOINT_RETAIN_COUNT = "checkpoint.retain.count"; + public static int DEFAULT_CHECKPOINT_RETAIN_COUNT = 5; + + public static final String ACTION_CHECKPOINT_INTERVAL = "action.checkpoint.interval"; + public static final long DEFAULT_ACTION_CHECKPOINT_INTERVAL = 10_000; + + public static final String MESSAGE_CHECKPOINT_INTERVAL = "message.checkpoint.interval"; + public static final long DEFAULT_MESSAGE_CHECKPOINT_INTERVAL = 10_000; +} diff --git a/qmq-server-common/src/main/java/qunar/tc/qmq/meta/BrokerAcquireMetaRequest.java b/qmq-server-common/src/main/java/qunar/tc/qmq/meta/BrokerAcquireMetaRequest.java new file mode 100644 index 00000000..3981ac64 --- /dev/null +++ b/qmq-server-common/src/main/java/qunar/tc/qmq/meta/BrokerAcquireMetaRequest.java @@ -0,0 +1,50 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.meta; + +/** + * @author keli.wang + * @since 2017/9/26 + */ +public class BrokerAcquireMetaRequest { + private String hostname; + private int port; + + public String getHostname() { + return hostname; + } + + public void setHostname(String hostname) { + this.hostname = hostname; + } + + public int getPort() { + return port; + } + + public void setPort(int port) { + this.port = port; + } + + @Override + public String toString() { + return "BrokerAcquireMetaRequest{" + + "hostname='" + hostname + '\'' + + ", port=" + port + + '}'; + } +} diff --git a/qmq-server-common/src/main/java/qunar/tc/qmq/meta/BrokerAcquireMetaRequestSerializer.java b/qmq-server-common/src/main/java/qunar/tc/qmq/meta/BrokerAcquireMetaRequestSerializer.java new file mode 100644 index 00000000..531bac06 --- /dev/null +++ b/qmq-server-common/src/main/java/qunar/tc/qmq/meta/BrokerAcquireMetaRequestSerializer.java @@ -0,0 +1,34 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.meta; + +import io.netty.buffer.ByteBuf; +import qunar.tc.qmq.utils.PayloadHolderUtils; + +public class BrokerAcquireMetaRequestSerializer { + public static void serialize(BrokerAcquireMetaRequest request, ByteBuf out) { + PayloadHolderUtils.writeString(request.getHostname(), out); + out.writeInt(request.getPort()); + } + + public static BrokerAcquireMetaRequest deSerialize(ByteBuf out) { + BrokerAcquireMetaRequest request = new BrokerAcquireMetaRequest(); + request.setHostname(PayloadHolderUtils.readString(out)); + request.setPort(out.readInt()); + return request; + } +} diff --git a/qmq-server-common/src/main/java/qunar/tc/qmq/meta/BrokerAcquireMetaResponse.java b/qmq-server-common/src/main/java/qunar/tc/qmq/meta/BrokerAcquireMetaResponse.java new file mode 100644 index 00000000..ab6d3e1b --- /dev/null +++ b/qmq-server-common/src/main/java/qunar/tc/qmq/meta/BrokerAcquireMetaResponse.java @@ -0,0 +1,60 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.meta; + +/** + * @author keli.wang + * @since 2017/9/26 + */ +public class BrokerAcquireMetaResponse { + private String name; + private BrokerRole role; + private String master; + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public BrokerRole getRole() { + return role; + } + + public void setRole(BrokerRole role) { + this.role = role; + } + + public String getMaster() { + return master; + } + + public void setMaster(String master) { + this.master = master; + } + + @Override + public String toString() { + return "BrokerAcquireMetaResponse{" + + "name='" + name + '\'' + + ", role=" + role + + ", master='" + master + '\'' + + '}'; + } +} diff --git a/qmq-server-common/src/main/java/qunar/tc/qmq/meta/BrokerAcquireMetaResponseSerializer.java b/qmq-server-common/src/main/java/qunar/tc/qmq/meta/BrokerAcquireMetaResponseSerializer.java new file mode 100644 index 00000000..c83f01a4 --- /dev/null +++ b/qmq-server-common/src/main/java/qunar/tc/qmq/meta/BrokerAcquireMetaResponseSerializer.java @@ -0,0 +1,36 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.meta; + +import io.netty.buffer.ByteBuf; +import qunar.tc.qmq.utils.PayloadHolderUtils; + +public class BrokerAcquireMetaResponseSerializer { + public static void serialize(BrokerAcquireMetaResponse response, ByteBuf out) { + PayloadHolderUtils.writeString(response.getName(), out); + PayloadHolderUtils.writeString(response.getMaster(), out); + out.writeInt(response.getRole().getCode()); + } + + public static BrokerAcquireMetaResponse deSerialize(ByteBuf out) { + BrokerAcquireMetaResponse response = new BrokerAcquireMetaResponse(); + response.setName(PayloadHolderUtils.readString(out)); + response.setMaster(PayloadHolderUtils.readString(out)); + response.setRole(BrokerRole.fromCode(out.readInt())); + return response; + } +} diff --git a/qmq-server-common/src/main/java/qunar/tc/qmq/meta/BrokerRegisterRequest.java b/qmq-server-common/src/main/java/qunar/tc/qmq/meta/BrokerRegisterRequest.java new file mode 100644 index 00000000..36cf344c --- /dev/null +++ b/qmq-server-common/src/main/java/qunar/tc/qmq/meta/BrokerRegisterRequest.java @@ -0,0 +1,80 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.meta; + +/** + * @author yunfeng.yang + * @since 2017/8/30 + */ +public class BrokerRegisterRequest { + private String groupName; + private int brokerRole; + private int brokerState; + private int requestType; + private String brokerAddress; + + public String getGroupName() { + return groupName; + } + + public void setGroupName(String groupName) { + this.groupName = groupName; + } + + public int getRequestType() { + return requestType; + } + + public void setRequestType(int requestType) { + this.requestType = requestType; + } + + public int getBrokerRole() { + return brokerRole; + } + + public void setBrokerRole(int brokerRole) { + this.brokerRole = brokerRole; + } + + public int getBrokerState() { + return brokerState; + } + + public void setBrokerState(int brokerState) { + this.brokerState = brokerState; + } + + public String getBrokerAddress() { + return brokerAddress; + } + + public void setBrokerAddress(String brokerAddress) { + this.brokerAddress = brokerAddress; + } + + @Override + public String toString() { + return "BrokerRegisterRequest{" + + "groupName='" + groupName + '\'' + + ", brokerRole=" + brokerRole + + ", brokerState=" + brokerState + + ", requestType=" + requestType + + ", brokerAddress='" + brokerAddress + '\'' + + '}'; + } +} diff --git a/qmq-server-common/src/main/java/qunar/tc/qmq/meta/BrokerRegisterRequestSerializer.java b/qmq-server-common/src/main/java/qunar/tc/qmq/meta/BrokerRegisterRequestSerializer.java new file mode 100644 index 00000000..e070e043 --- /dev/null +++ b/qmq-server-common/src/main/java/qunar/tc/qmq/meta/BrokerRegisterRequestSerializer.java @@ -0,0 +1,40 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.meta; + +import io.netty.buffer.ByteBuf; +import qunar.tc.qmq.utils.PayloadHolderUtils; + +public class BrokerRegisterRequestSerializer { + public static void serialize(BrokerRegisterRequest request, ByteBuf out) { + out.writeInt(request.getRequestType()); + PayloadHolderUtils.writeString(request.getGroupName(), out); + PayloadHolderUtils.writeString(request.getBrokerAddress(), out); + out.writeInt(request.getBrokerRole()); + out.writeInt(request.getBrokerState()); + } + + public static BrokerRegisterRequest deSerialize(ByteBuf out) { + BrokerRegisterRequest request = new BrokerRegisterRequest(); + request.setRequestType(out.readInt()); + request.setGroupName(PayloadHolderUtils.readString(out)); + request.setBrokerAddress(PayloadHolderUtils.readString(out)); + request.setBrokerRole(out.readInt()); + request.setBrokerState(out.readInt()); + return request; + } +} diff --git a/qmq-server-common/src/main/java/qunar/tc/qmq/meta/BrokerRegisterResponse.java b/qmq-server-common/src/main/java/qunar/tc/qmq/meta/BrokerRegisterResponse.java new file mode 100644 index 00000000..292b8dcb --- /dev/null +++ b/qmq-server-common/src/main/java/qunar/tc/qmq/meta/BrokerRegisterResponse.java @@ -0,0 +1,35 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.meta; + +import java.util.List; + +/** + * @author yunfeng.yang + * @since 2017/9/1 + */ +public class BrokerRegisterResponse { + private List subjects; + + public List getSubjects() { + return subjects; + } + + public void setSubjects(List subjects) { + this.subjects = subjects; + } +} diff --git a/qmq-server-common/src/main/java/qunar/tc/qmq/meta/BrokerRegisterService.java b/qmq-server-common/src/main/java/qunar/tc/qmq/meta/BrokerRegisterService.java new file mode 100644 index 00000000..644ab592 --- /dev/null +++ b/qmq-server-common/src/main/java/qunar/tc/qmq/meta/BrokerRegisterService.java @@ -0,0 +1,194 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.meta; + +import com.google.common.base.Optional; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import qunar.tc.qmq.common.Disposable; +import qunar.tc.qmq.concurrent.NamedThreadFactory; +import qunar.tc.qmq.configuration.BrokerConfig; +import qunar.tc.qmq.netty.NettyClientConfig; +import qunar.tc.qmq.netty.client.NettyClient; +import qunar.tc.qmq.protocol.CommandCode; +import qunar.tc.qmq.protocol.Datagram; +import qunar.tc.qmq.protocol.RemotingHeader; +import qunar.tc.qmq.utils.NetworkUtils; + +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; + +/** + * @author keli.wang + * @since 2017/9/1 + */ +public class BrokerRegisterService implements Disposable { + private static final Logger LOG = LoggerFactory.getLogger(BrokerRegisterService.class); + + private static final long TIMEOUT_MS = TimeUnit.SECONDS.toMillis(5); + private static final int HEARTBEAT_DELAY_SECONDS = 10; + + private final ScheduledExecutorService heartbeatScheduler; + private final MetaServerLocator locator; + private final NettyClient client; + private final int port; + private final String brokerAddress; + + private volatile int brokerState; + private volatile String endpoint; + + public BrokerRegisterService(final int port, final MetaServerLocator locator) { + this.heartbeatScheduler = Executors.newSingleThreadScheduledExecutor(new NamedThreadFactory("broker-register-heartbeat")); + this.locator = locator; + this.client = NettyClient.getClient(); + this.client.start(new NettyClientConfig()); + this.port = port; + this.brokerAddress = BrokerConfig.getBrokerAddress() + ":" + port; + this.brokerState = BrokerState.NRW.getCode(); + + repickEndpoint(); + } + + public void start() { + acquireMeta(); + heartbeatScheduler.scheduleWithFixedDelay(this::heartbeat, 0, HEARTBEAT_DELAY_SECONDS, TimeUnit.SECONDS); + } + + private void acquireMeta() { + Datagram datagram = null; + try { + datagram = client.sendSync(endpoint, buildAcquireMetaDatagram(), TIMEOUT_MS); + final BrokerAcquireMetaResponse meta = BrokerAcquireMetaResponseSerializer.deSerialize(datagram.getBody()); + BrokerConfig.getInstance().updateMeta(meta); + } catch (Exception e) { + LOG.error("Send acquire meta message to meta server failed", e); + throw new RuntimeException(e); + } finally { + if (datagram != null) { + datagram.release(); + } + } + } + + private void heartbeat() { + Datagram datagram = null; + try { + datagram = client.sendSync(endpoint, buildRegisterDatagram(BrokerRequestType.HEARTBEAT), TIMEOUT_MS); + } catch (Exception e) { + LOG.error("Send HEARTBEAT message to meta server failed", e); + repickEndpoint(); + } finally { + if (datagram != null) { + datagram.release(); + } + } + } + + public void healthSwitch(final Boolean online) { + if (online) { + brokerOnline(); + } else { + brokerOffline(); + } + } + + private void brokerOnline() { + Datagram datagram = null; + try { + brokerState = BrokerState.RW.getCode(); + datagram = client.sendSync(endpoint, buildRegisterDatagram(BrokerRequestType.ONLINE), TIMEOUT_MS); + } catch (Exception e) { + LOG.error("Send ONLINE message to meta server failed", e); + repickEndpoint(); + throw new RuntimeException("broker online failed", e); + } finally { + if (datagram != null) { + datagram.release(); + } + } + } + + private void brokerOffline() { + Datagram datagram = null; + try { + brokerState = BrokerState.NRW.getCode(); + datagram = client.sendSync(endpoint, buildRegisterDatagram(BrokerRequestType.OFFLINE), TIMEOUT_MS); + } catch (Exception e) { + LOG.error("Send OFFLINE message to meta server failed", e); + repickEndpoint(); + throw new RuntimeException("broker offline failed", e); + } finally { + if (datagram != null) { + datagram.release(); + } + } + } + + private void repickEndpoint() { + Optional optional = locator.queryEndpoint(); + if (optional.isPresent()) { + this.endpoint = optional.get(); + } + } + + private Datagram buildAcquireMetaDatagram() { + final Datagram datagram = new Datagram(); + final RemotingHeader header = new RemotingHeader(); + header.setCode(CommandCode.BROKER_ACQUIRE_META); + datagram.setHeader(header); + datagram.setPayloadHolder(out -> { + final BrokerAcquireMetaRequest request = new BrokerAcquireMetaRequest(); + request.setHostname(NetworkUtils.getLocalHostname()); + request.setPort(port); + BrokerAcquireMetaRequestSerializer.serialize(request, out); + }); + return datagram; + } + + private Datagram buildRegisterDatagram(final BrokerRequestType checkType) { + final Datagram datagram = new Datagram(); + final RemotingHeader header = new RemotingHeader(); + header.setCode(CommandCode.BROKER_REGISTER); + datagram.setHeader(header); + datagram.setPayloadHolder(out -> { + final BrokerRegisterRequest request = buildRegisterRequest(checkType); + BrokerRegisterRequestSerializer.serialize(request, out); + }); + return datagram; + } + + private BrokerRegisterRequest buildRegisterRequest(final BrokerRequestType checkType) { + final BrokerRegisterRequest request = new BrokerRegisterRequest(); + request.setGroupName(BrokerConfig.getBrokerName()); + request.setBrokerRole(BrokerConfig.getBrokerRole().getCode()); + request.setBrokerState(brokerState); + request.setRequestType(checkType.getCode()); + request.setBrokerAddress(brokerAddress); + return request; + } + + @Override + public void destroy() { + heartbeatScheduler.shutdown(); + try { + heartbeatScheduler.awaitTermination(10, TimeUnit.SECONDS); + } catch (InterruptedException e) { + LOG.error("Shutdown heartbeat scheduler interrupted."); + } + } +} diff --git a/qmq-server-common/src/main/java/qunar/tc/qmq/meta/BrokerRequestType.java b/qmq-server-common/src/main/java/qunar/tc/qmq/meta/BrokerRequestType.java new file mode 100644 index 00000000..1686c798 --- /dev/null +++ b/qmq-server-common/src/main/java/qunar/tc/qmq/meta/BrokerRequestType.java @@ -0,0 +1,35 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.meta; + +/** + * @author yunfeng.yang + * @since 2017/8/31 + */ +public enum BrokerRequestType { + ONLINE(0), HEARTBEAT(1), OFFLINE(2); + + private int code; + + BrokerRequestType(int code) { + this.code = code; + } + + public int getCode() { + return code; + } +} diff --git a/qmq-server-common/src/main/java/qunar/tc/qmq/meta/BrokerRole.java b/qmq-server-common/src/main/java/qunar/tc/qmq/meta/BrokerRole.java new file mode 100644 index 00000000..f53b6c2e --- /dev/null +++ b/qmq-server-common/src/main/java/qunar/tc/qmq/meta/BrokerRole.java @@ -0,0 +1,58 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.meta; + +/** + * @author yunfeng.yang + * @since 2017/8/19 + */ +public enum BrokerRole { + MASTER(BrokerGroupKind.NORMAL, 0), + SLAVE(BrokerGroupKind.NORMAL, 1), + STANDBY(BrokerGroupKind.NORMAL, 2), + DELAY(BrokerGroupKind.DELAY, 3), + BACKUP(BrokerGroupKind.NORMAL, 4), + DELAY_MASTER(BrokerGroupKind.DELAY, 5), + DELAY_SLAVE(BrokerGroupKind.DELAY, 6), + DELAY_BACKUP(BrokerGroupKind.DELAY, 7); + + private final BrokerGroupKind kind; + private final int code; + + BrokerRole(final BrokerGroupKind kind, final int code) { + this.kind = kind; + this.code = code; + } + + public static BrokerRole fromCode(int role) { + for (BrokerRole value : BrokerRole.values()) { + if (value.getCode() == role) { + return value; + } + } + + throw new RuntimeException("Unknown broker role code " + role); + } + + public BrokerGroupKind getKind() { + return kind; + } + + public int getCode() { + return code; + } +} diff --git a/qmq-server-common/src/main/java/qunar/tc/qmq/monitor/QMon.java b/qmq-server-common/src/main/java/qunar/tc/qmq/monitor/QMon.java new file mode 100644 index 00000000..e795f049 --- /dev/null +++ b/qmq-server-common/src/main/java/qunar/tc/qmq/monitor/QMon.java @@ -0,0 +1,393 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.monitor; + +import com.google.common.base.Supplier; +import qunar.tc.qmq.metrics.Metrics; + +import java.util.concurrent.TimeUnit; + +import static qunar.tc.qmq.metrics.MetricsConstants.SUBJECT_ARRAY; +import static qunar.tc.qmq.metrics.MetricsConstants.SUBJECT_GROUP_ARRAY; + +/** + * @author yunfeng.yang + * @since 2017/7/10 + */ +public final class QMon { + private static final String[] NONE = {}; + private static final String[] CONSUMER_ID = {"consumerId"}; + private static final String[] REQUEST_CODE = {"requestCode"}; + + private static void subjectCountInc(String name, String subject) { + countInc(name, SUBJECT_ARRAY, new String[]{subject}); + } + + private static void subjectAndGroupCountInc(String name, String subjectPrefix, String consumerGroup) { + countInc(name, SUBJECT_GROUP_ARRAY, new String[]{subjectPrefix, consumerGroup}); + } + + private static void subjectAndGroupCountInc(String name, String[] values, long num) { + countInc(name, SUBJECT_GROUP_ARRAY, values, num); + } + + private static void countInc(String name, String[] tags, String[] values) { + Metrics.counter(name, tags, values).inc(); + } + + private static void countDec(String name, String[] tags, String[] values) { + Metrics.counter(name, tags, values).dec(); + } + + private static void countInc(String name, String[] tags, String[] values, long num) { + Metrics.counter(name, tags, values).inc(num); + } + + public static void produceTime(String subject, long time) { + Metrics.timer("produceTime", SUBJECT_ARRAY, new String[]{subject}).update(time, TimeUnit.MILLISECONDS); + } + + public static void receivedMessagesCountInc(String subject) { + subjectCountInc("receivedMessagesCount", subject); + final String[] values = {subject}; + Metrics.meter("receivedMessagesEx", SUBJECT_ARRAY, values).mark(); + } + + public static void receivedIllegalSubjectMessagesCountInc(String subject) { + subjectCountInc("receivedIllegalSubjectMessagesCount", subject); + } + + public static void pulledMessagesCountInc(String subject, String group, int messageNum) { + final String[] values = new String[]{subject, group}; + subjectAndGroupCountInc("pulledMessagesCount", values, messageNum); + Metrics.meter("pulledMessagesEx", SUBJECT_GROUP_ARRAY, values).mark(messageNum); + } + + public static void pulledNoMessagesCountInc(String subject, String group) { + subjectAndGroupCountInc("pulledNoMessagesCount", subject, group); + } + + public static void pulledMessageBytesCountInc(String subject, String group, int bytes) { + subjectAndGroupCountInc("pulledMessageBytesCount", new String[]{subject, group}, bytes); + } + + public static void storeMessageErrorCountInc(String subject) { + subjectCountInc("storeMessageErrorCount", subject); + } + + public static void pullQueueTime(String subject, String group, long start) { + Metrics.timer("pullQueueTime", SUBJECT_GROUP_ARRAY, new String[]{subject, group}).update(System.currentTimeMillis() - start, TimeUnit.MILLISECONDS); + } + + public static void suspendRequestCountInc(String subject, String group) { + subjectAndGroupCountInc("suspendRequestCount", subject, group); + } + + public static void resumeActorCountInc(String subject, String group) { + subjectAndGroupCountInc("resumeActorCount", subject, group); + } + + public static void pullTimeOutCountInc(String subject, String group) { + subjectAndGroupCountInc("pullTimeOutCount", subject, group); + } + + public static void pullExpiredCountInc(String subject, String group) { + subjectAndGroupCountInc("pullExpiredCount", subject, group); + } + + public static void getMessageErrorCountInc(String subject, String group) { + subjectAndGroupCountInc("getMessageErrorCount", subject, group); + } + + public static void getMessageOverflowCountInc(String subject, String group) { + subjectAndGroupCountInc("getMessageOverflowCount", subject, group); + } + + public static void consumerAckCountInc(String subject, String group, int size) { + subjectAndGroupCountInc("consumerAckCount", new String[]{subject, group}, size); + } + + public static void consumerLostAckCountInc(String subject, String group, int lostAckCount) { + subjectAndGroupCountInc("consumerLostAckCount", new String[]{subject, group}, lostAckCount); + } + + public static void consumerDuplicateAckCountInc(String subject, String group, int duplicateAckCount) { + subjectAndGroupCountInc("consumerDuplicateAckCount", new String[]{subject, group}, duplicateAckCount); + } + + public static void consumerAckTimeoutErrorCountInc(String consumerId, int num) { + countInc("consumerAckTimeoutErrorCountInc", CONSUMER_ID, new String[]{consumerId}, num); + } + + public static void putMessageTime(String subject, long time) { + Metrics.timer("putMessageTime", SUBJECT_ARRAY, new String[]{subject}).update(time, TimeUnit.MILLISECONDS); + } + + public static void processTime(String subject, long time) { + Metrics.timer("processTime", SUBJECT_ARRAY, new String[]{subject}).update(time, TimeUnit.MILLISECONDS); + } + + public static void rejectReceivedMessageCountInc(String subject) { + subjectCountInc("rejectReceivedMessageCount", subject); + } + + public static void brokerReadOnlyMessageCountInc(String subject) { + subjectCountInc("brokerReadOnlyMessageCount", subject); + } + + public static void receivedFailedCountInc(String subject) { + subjectCountInc("receivedFailedCount", subject); + } + + public static void receiveExpiredMessagesCountInc(String subject) { + subjectCountInc("receiveExpiredMessagesCount", subject); + } + + public static void ackProcessTime(String subject, String group, long elapsed) { + Metrics.timer("ackProcessTime", SUBJECT_GROUP_ARRAY, new String[]{subject, group}).update(elapsed, TimeUnit.MILLISECONDS); + } + + public static void pullProcessTime(String subject, String group, long elapsed) { + Metrics.timer("pullProcessTime", SUBJECT_GROUP_ARRAY, new String[]{subject, group}).update(elapsed, TimeUnit.MILLISECONDS); + } + + public static void putActionFailedCountInc(String subject, String group) { + subjectAndGroupCountInc("putActionFailedCount", subject, group); + } + + public static void findLostMessageCountInc(String subject, String group, int messageNum) { + subjectAndGroupCountInc("findLostMessageCount", new String[]{subject, group}, messageNum); + } + + public static void findLostMessageEmptyCountInc(String subject, String group) { + subjectAndGroupCountInc("findLostMessageEmptyCount", subject, group); + } + + public static void pullRequestCountInc(String subject, String group) { + subjectAndGroupCountInc("pullRequestCount", subject, group); + Metrics.meter("pullRequestEx", SUBJECT_GROUP_ARRAY, new String[]{subject, group}).mark(); + } + + public static void ackRequestCountInc(String subject, String group) { + subjectAndGroupCountInc("ackRequestCount", subject, group); + Metrics.meter("ackRequestEx", SUBJECT_GROUP_ARRAY, new String[]{subject, group}).mark(); + } + + public static void pullParamErrorCountInc(String subject, String group) { + subjectAndGroupCountInc("pullParamErrorCount", subject, group); + } + + public static void nonPositiveRequestNumCountInc(String subject, String group) { + subjectAndGroupCountInc("nonPositiveRequestNumCount", subject, group); + } + + public static void putAckActionsErrorCountInc(String subject, String group) { + subjectAndGroupCountInc("putAckActionsErrorCount", subject, group); + } + + public static void findMessagesErrorCountInc(String subject, String group) { + subjectAndGroupCountInc("findMessagesErrorCount", subject, group); + } + + public static void putNeedRetryMessagesCountInc(String subject, String group, int size) { + subjectAndGroupCountInc("putNeedRetryMessagesCount", new String[]{subject, group}, size); + } + + public static void consumerLogOffsetRangeError(String subject, String group) { + subjectAndGroupCountInc("consumerLogOffsetRangeErrorCount", subject, group); + } + + public static void consumerErrorCount(String subject, String group) { + subjectAndGroupCountInc("consumerErrorCount", subject, group); + } + + public static void deadLetterQueueCount(String subject, String group) { + subjectAndGroupCountInc("deadLetterQueueCount", subject, group); + } + + public static void expiredMessagesCountInc(String subject, String group, long num) { + subjectAndGroupCountInc("ExpiredMessages", new String[]{subject, group}, num); + } + + public static void readMessageReturnNullCountInc(String subject) { + subjectCountInc("ReadMessageReturnNull", subject); + } + + public static void replayMessageLogFailedCountInc() { + countInc("ReplayMessageLogFailed", NONE, NONE); + } + + public static void replayActionLogFailedCountInc() { + countInc("ReplayActionLogFailed", NONE, NONE); + } + + public static void logSegmentTotalRefCountInc() { + countInc("LogSegment.TotalRefCount", NONE, NONE); + } + + public static void logSegmentTotalRefCountDec() { + countDec("LogSegment.TotalRefCount", NONE, NONE); + } + + public static void maybeLostMessagesCountInc(String subject, String group, long num) { + subjectAndGroupCountInc("maybeLostMessages", new String[]{subject, group}, num); + } + + public static void retryTaskExecuteCountInc(String subject, String group) { + subjectAndGroupCountInc("ConsumerStatusChecker.RetryTask.Execute", subject, group); + } + + public static void offlineTaskExecuteCountInc(String subject, String group) { + subjectAndGroupCountInc("ConsumerStatusChecker.OfflineTask.Execute", subject, group); + } + + public static void brokerReceivedInvalidMessageCountInc() { + countInc("Broker.ReceivedInvalidMessage", NONE, NONE); + } + + public static void findNewExistMessageTime(String subject, String group, long elapsedMillis) { + Metrics.timer("findNewExistMessagesTime", SUBJECT_GROUP_ARRAY, new String[]{subject, group}).update(elapsedMillis, TimeUnit.MILLISECONDS); + } + + public static void findLostMessagesTime(String subject, String group, long elapsedMillis) { + Metrics.timer("findLostMessagesTime", SUBJECT_GROUP_ARRAY, new String[]{subject, group}).update(elapsedMillis, TimeUnit.MILLISECONDS); + } + + public static void readPullResultAsBytesElapsed(String subject, String group, long elapsedMillis) { + Metrics.timer("Broker.PullResult.ReadAsBytesElapsed", SUBJECT_GROUP_ARRAY, new String[]{subject, group}).update(elapsedMillis, TimeUnit.MILLISECONDS); + } + + public static void flushPullLogCountInc() { + countInc("Store.PullLog.FlushCount", NONE, NONE); + } + + public static void flushConsumerLogCountInc(String subject) { + subjectCountInc("Store.ConsumerLog.FlushCount", subject); + } + + public static void flushPullLogTimer(long elapsedMillis) { + Metrics.timer("Store.PullLog.FlushTimer", NONE, NONE).update(elapsedMillis, TimeUnit.MILLISECONDS); + } + + public static void flushActionLogTimer(long elapsedMillis) { + Metrics.timer("Store.MessageLog.FlushTimer", NONE, NONE).update(elapsedMillis, TimeUnit.MILLISECONDS); + } + + public static void flushConsumerLogTimer(long elapsedMillis) { + Metrics.timer("Store.ConsumerLog.FlushTimer", NONE, NONE).update(elapsedMillis, TimeUnit.MILLISECONDS); + } + + public static void flushMessageLogTimer(long elapsedMillis) { + Metrics.timer("Store.MessageLog.FlushTimer", NONE, NONE).update(elapsedMillis, TimeUnit.MILLISECONDS); + } + + public static void pullLogFlusherExceedCheckpointIntervalCountInc() { + countInc("PullLogFlusher.ExceedCheckpointInterval", NONE, NONE); + } + + public static void pullLogFlusherElapsedPerExecute(long elapsedMillis) { + Metrics.timer("PullLogFlusher.ElapsedPerExecute", NONE, NONE).update(elapsedMillis, TimeUnit.MILLISECONDS); + } + + public static void pullLogFlusherFlushFailedCountInc() { + countInc("PullLogFlusher.FlushFailed", NONE, NONE); + } + + public static void consumerLogFlusherExceedCheckpointIntervalCountInc() { + countInc("ConsumerLogFlusher.ExceedCheckpointInterval", NONE, NONE); + } + + public static void consumerLogFlusherElapsedPerExecute(long elapsedMillis) { + Metrics.timer("ConsumerLogFlusher.ElapsedPerExecute", NONE, NONE).update(elapsedMillis, TimeUnit.MILLISECONDS); + } + + public static void consumerLogFlusherFlushFailedCountInc() { + countInc("ConsumerLogFlusher.FlushFailed", NONE, NONE); + } + + public static void hitDeletedConsumerLogSegmentCountInc(String subject) { + subjectCountInc("HitDeletedConsumerLogSegment", subject); + } + + public static void adjustConsumerLogMinOffset(String subject) { + subjectCountInc("ConsumerLog.AdjustMinOffset", subject); + } + + public static void nettyRequestExecutorExecuteTimer(long elapsedMillis) { + Metrics.timer("NettyRequestExecutor.execute", NONE, NONE).update(elapsedMillis, TimeUnit.MILLISECONDS); + } + + public static void executorQueueSizeGauge(String requestCode, Supplier supplier) { + Metrics.gauge("executorQueueSize", REQUEST_CODE, new String[]{requestCode}, supplier); + } + + public static void activeConnectionGauge(String name, Supplier supplier) { + Metrics.gauge("NettyProvider.Connection.ActiveCount", new String[]{"name"}, new String[]{name}, supplier); + } + + public static void activeClientCount(Supplier supplier) { + Metrics.gauge("Broker.ActiveClientCount", NONE, NONE, supplier); + } + + public static void messageSequenceLagGauge(String subject, String group, Supplier supplier) { + Metrics.gauge("messageSequenceLag", SUBJECT_GROUP_ARRAY, new String[]{subject, group}, supplier); + } + + public static void replayMessageLogLag(Supplier supplier) { + Metrics.gauge("ReplayMessageLogLag", NONE, NONE, supplier); + } + + public static void replayActionLogLag(Supplier supplier) { + Metrics.gauge("ReplayActionLogLag", NONE, NONE, supplier); + } + + public static void slaveMessageLogLagGauge(String role, Supplier supplier) { + Metrics.gauge("slaveMessageLogLag", new String[]{"role"}, new String[]{role}, supplier); + } + + public static void slaveActionLogLagGauge(String role, Supplier supplier) { + Metrics.gauge("slaveActionLogLag", new String[]{"role"}, new String[]{role}, supplier); + } + + public static void removeMessageSequenceLag(String subject, String group) { + Metrics.remove("messageSequenceLag", SUBJECT_GROUP_ARRAY, new String[]{subject, group}); + } + + public static void syncTaskExecTimer(String processor, long elapsedMillis) { + Metrics.timer("SyncTask.ExecTimer", new String[]{"processor"}, new String[]{processor}).update(elapsedMillis, TimeUnit.MILLISECONDS); + } + + public static void syncTaskSyncFailedCountInc(String brokerGroup) { + countInc("SyncTask.SyncFailedCount", new String[]{"BrokerGroup"}, new String[]{brokerGroup}); + } + + public static void dispatchersGauge(String name, Supplier supplier) { + Metrics.gauge("dispatchers-" + name, NONE, NONE, supplier); + } + + public static void actorQueueGauge(String systemName, String name, Supplier supplier) { + Metrics.gauge("actor-queue-" + systemName, new String[]{"name"}, new String[]{name}, supplier); + } + + public static void actorSystemQueueGauge(String name, Supplier supplier) { + Metrics.gauge("actorsystem-queue-" + name, NONE, NONE, supplier); + } + + public static void actorProcessTime(String actor, long elapsedMillis) { + Metrics.timer("actor-processTime", new String[]{"actor"}, new String[]{actor}).update(elapsedMillis, TimeUnit.MILLISECONDS); + } +} diff --git a/qmq-server-common/src/main/java/qunar/tc/qmq/netty/ConnectionEventHandler.java b/qmq-server-common/src/main/java/qunar/tc/qmq/netty/ConnectionEventHandler.java new file mode 100644 index 00000000..972d14fd --- /dev/null +++ b/qmq-server-common/src/main/java/qunar/tc/qmq/netty/ConnectionEventHandler.java @@ -0,0 +1,29 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.netty; + +import io.netty.channel.ChannelHandlerContext; + +/** + * @author keli.wang + * @since 2018/7/18 + */ +public interface ConnectionEventHandler { + void channelActive(ChannelHandlerContext ctx); + + void channelInactive(ChannelHandlerContext ctx); +} diff --git a/qmq-server-common/src/main/java/qunar/tc/qmq/netty/ConnectionHandler.java b/qmq-server-common/src/main/java/qunar/tc/qmq/netty/ConnectionHandler.java new file mode 100644 index 00000000..b59ccb33 --- /dev/null +++ b/qmq-server-common/src/main/java/qunar/tc/qmq/netty/ConnectionHandler.java @@ -0,0 +1,46 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.netty; + +import io.netty.channel.ChannelHandler; +import io.netty.channel.ChannelHandlerContext; +import io.netty.channel.ChannelInboundHandlerAdapter; + +/** + * @author keli.wang + * @since 2017/2/28 + */ +@ChannelHandler.Sharable +class ConnectionHandler extends ChannelInboundHandlerAdapter { + private final ConnectionEventHandler handler; + + ConnectionHandler(final ConnectionEventHandler handler) { + this.handler = handler; + } + + @Override + public void channelActive(ChannelHandlerContext ctx) throws Exception { + handler.channelActive(ctx); + super.channelActive(ctx); + } + + @Override + public void channelInactive(ChannelHandlerContext ctx) throws Exception { + handler.channelInactive(ctx); + super.channelInactive(ctx); + } +} diff --git a/qmq-server-common/src/main/java/qunar/tc/qmq/netty/DefaultConnectionEventHandler.java b/qmq-server-common/src/main/java/qunar/tc/qmq/netty/DefaultConnectionEventHandler.java new file mode 100644 index 00000000..6632b5cd --- /dev/null +++ b/qmq-server-common/src/main/java/qunar/tc/qmq/netty/DefaultConnectionEventHandler.java @@ -0,0 +1,53 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.netty; + +import io.netty.channel.ChannelHandlerContext; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import qunar.tc.qmq.monitor.QMon; + +import java.util.concurrent.atomic.AtomicLong; + +/** + * @author keli.wang + * @since 2018/7/18 + */ +public class DefaultConnectionEventHandler implements ConnectionEventHandler { + private static final Logger LOG = LoggerFactory.getLogger(ConnectionHandler.class); + + private final String name; + private final AtomicLong counter = new AtomicLong(0); + + public DefaultConnectionEventHandler(final String name) { + this.name = name; + QMon.activeConnectionGauge(name, counter::doubleValue); + } + + @Override + public void channelActive(ChannelHandlerContext ctx) { + LOG.info("[name: {}] client {} connected", name, ctx.channel().remoteAddress()); + counter.incrementAndGet(); + + } + + @Override + public void channelInactive(ChannelHandlerContext ctx) { + LOG.info("[name: {}] client {} disconnected", name, ctx.channel().remoteAddress()); + counter.decrementAndGet(); + } +} diff --git a/qmq-server-common/src/main/java/qunar/tc/qmq/netty/NettyRequestExecutor.java b/qmq-server-common/src/main/java/qunar/tc/qmq/netty/NettyRequestExecutor.java new file mode 100644 index 00000000..d17f5960 --- /dev/null +++ b/qmq-server-common/src/main/java/qunar/tc/qmq/netty/NettyRequestExecutor.java @@ -0,0 +1,110 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.netty; + +import io.netty.channel.ChannelHandlerContext; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import qunar.tc.qmq.monitor.QMon; +import qunar.tc.qmq.protocol.CommandCode; +import qunar.tc.qmq.protocol.Datagram; +import qunar.tc.qmq.protocol.RemotingCommand; +import qunar.tc.qmq.protocol.RemotingHeader; +import qunar.tc.qmq.util.RemotingBuilder; + +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.RejectedExecutionException; +import java.util.concurrent.ThreadPoolExecutor; + +/** + * @author yunfeng.yang + * @since 2017/7/3 + */ +class NettyRequestExecutor { + private static final Logger LOG = LoggerFactory.getLogger(NettyRequestExecutor.class); + + private final NettyRequestProcessor processor; + private final ExecutorService executor; + + NettyRequestExecutor(final short requestCode, final NettyRequestProcessor processor, final ExecutorService executor) { + this.processor = processor; + this.executor = executor; + if (executor != null) { + if (executor instanceof ThreadPoolExecutor) { + QMon.executorQueueSizeGauge(String.valueOf(requestCode), () -> (double) ((ThreadPoolExecutor) executor).getQueue().size()); + } + } + } + + void execute(final ChannelHandlerContext ctx, final RemotingCommand cmd) { + if (executor == null) { + executeWithMonitor(ctx, cmd); + return; + } + + try { + executor.execute(() -> executeWithMonitor(ctx, cmd)); + } catch (RejectedExecutionException e) { + ctx.writeAndFlush(errorResp(CommandCode.BROKER_ERROR, cmd)); + } + } + + private void executeWithMonitor(final ChannelHandlerContext ctx, RemotingCommand cmd) { + final long start = System.currentTimeMillis(); + try { + doExecute(ctx, cmd); + } finally { + cmd.release(); + QMon.nettyRequestExecutorExecuteTimer(System.currentTimeMillis() - start); + } + } + + private void doExecute(final ChannelHandlerContext ctx, final RemotingCommand cmd) { + final int opaque = cmd.getHeader().getOpaque(); + + if (processor.rejectRequest()) { + ctx.writeAndFlush(errorResp(CommandCode.BROKER_REJECT, cmd)); + return; + } + + try { + final CompletableFuture future = processor.processRequest(ctx, cmd); + if (cmd.isOneWay()) { + return; + } + + if (future != null) { + future.exceptionally(ex -> errorResp(CommandCode.BROKER_ERROR, cmd)) + .thenAccept((datagram -> { + final RemotingHeader header = datagram.getHeader(); + header.setOpaque(opaque); + header.setVersion(cmd.getHeader().getVersion()); + header.setRequestCode(cmd.getHeader().getCode()); + ctx.writeAndFlush(datagram); + })); + } + } catch (Throwable e) { + LOG.error("doExecute request exception, channel:{}, cmd:{}", ctx.channel(), cmd, e); + ctx.writeAndFlush(errorResp(CommandCode.BROKER_ERROR, cmd)); + } + } + + private Datagram errorResp(final short code, final RemotingCommand command) { + return RemotingBuilder.buildEmptyResponseDatagram(code, command.getHeader()); + } +} diff --git a/qmq-server-common/src/main/java/qunar/tc/qmq/netty/NettyRequestProcessor.java b/qmq-server-common/src/main/java/qunar/tc/qmq/netty/NettyRequestProcessor.java new file mode 100644 index 00000000..a0743eff --- /dev/null +++ b/qmq-server-common/src/main/java/qunar/tc/qmq/netty/NettyRequestProcessor.java @@ -0,0 +1,33 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.netty; + +import io.netty.channel.ChannelHandlerContext; +import qunar.tc.qmq.protocol.Datagram; +import qunar.tc.qmq.protocol.RemotingCommand; + +import java.util.concurrent.CompletableFuture; + +/** + * @author yunfeng.yang + * @since 2017/7/3 + */ +public interface NettyRequestProcessor { + CompletableFuture processRequest(ChannelHandlerContext ctx, RemotingCommand request); + + boolean rejectRequest(); +} diff --git a/qmq-server-common/src/main/java/qunar/tc/qmq/netty/NettyServer.java b/qmq-server-common/src/main/java/qunar/tc/qmq/netty/NettyServer.java new file mode 100644 index 00000000..f526dce5 --- /dev/null +++ b/qmq-server-common/src/main/java/qunar/tc/qmq/netty/NettyServer.java @@ -0,0 +1,97 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.netty; + +import io.netty.bootstrap.ServerBootstrap; +import io.netty.channel.Channel; +import io.netty.channel.ChannelInitializer; +import io.netty.channel.ChannelOption; +import io.netty.channel.nio.NioEventLoopGroup; +import io.netty.channel.socket.SocketChannel; +import io.netty.channel.socket.nio.NioServerSocketChannel; +import io.netty.util.concurrent.DefaultThreadFactory; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import qunar.tc.qmq.common.Disposable; + +import java.util.concurrent.ExecutorService; + +/** + * @author yunfeng.yang + * @since 2017/6/30 + */ +public class NettyServer implements Disposable { + private static final Logger LOG = LoggerFactory.getLogger(NettyServer.class); + + private final NioEventLoopGroup bossGroup; + private final NioEventLoopGroup workerGroup; + + private final ServerBootstrap bootstrap; + private final NettyServerHandler serverHandler; + private final ConnectionHandler connectionHandler; + + private final int port; + private volatile Channel channel; + + public NettyServer(final String name, final int workerCount, final int port, final ConnectionEventHandler connectionEventHandler) { + this.bossGroup = new NioEventLoopGroup(1, new DefaultThreadFactory(name + "-netty-server-boss", true)); + this.workerGroup = new NioEventLoopGroup(workerCount, new DefaultThreadFactory(name + "-netty-server-worker", true)); + this.port = port; + this.bootstrap = new ServerBootstrap(); + this.serverHandler = new NettyServerHandler(); + this.connectionHandler = new ConnectionHandler(connectionEventHandler); + } + + public void registerProcessor(final short requestCode, final NettyRequestProcessor processor) { + registerProcessor(requestCode, processor, null); + } + + public void registerProcessor(final short requestCode, + final NettyRequestProcessor processor, + final ExecutorService executorService) { + serverHandler.registerProcessor(requestCode, processor, executorService); + } + + public void start() { + bootstrap.option(ChannelOption.SO_REUSEADDR, true); + bootstrap.childOption(ChannelOption.TCP_NODELAY, true); + bootstrap.group(bossGroup, workerGroup) + .channel(NioServerSocketChannel.class) + .childHandler(new ChannelInitializer() { + @Override + protected void initChannel(SocketChannel ch) { + ch.pipeline().addLast("connectionHandler", connectionHandler); + ch.pipeline().addLast("encoder", new EncodeHandler()); + ch.pipeline().addLast("decoder", new DecodeHandler(true)); + ch.pipeline().addLast("dispatcher", serverHandler); + } + }); + try { + channel = bootstrap.bind(port).await().channel(); + } catch (InterruptedException e) { + LOG.error("server start fail", e); + } + LOG.info("listen on port {}", port); + } + + @Override + public void destroy() { + if (channel != null && channel.isActive()) { + channel.close().awaitUninterruptibly(); + } + } +} diff --git a/qmq-server-common/src/main/java/qunar/tc/qmq/netty/NettyServerHandler.java b/qmq-server-common/src/main/java/qunar/tc/qmq/netty/NettyServerHandler.java new file mode 100644 index 00000000..af50df20 --- /dev/null +++ b/qmq-server-common/src/main/java/qunar/tc/qmq/netty/NettyServerHandler.java @@ -0,0 +1,82 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.netty; + +import com.google.common.collect.Maps; +import io.netty.channel.ChannelHandler; +import io.netty.channel.ChannelHandlerContext; +import io.netty.channel.SimpleChannelInboundHandler; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import qunar.tc.qmq.protocol.CommandCode; +import qunar.tc.qmq.protocol.Datagram; +import qunar.tc.qmq.protocol.RemotingCommand; +import qunar.tc.qmq.util.RemotingBuilder; + +import java.util.Map; +import java.util.concurrent.ExecutorService; + +/** + * @author yunfeng.yang + * @since 2017/7/3 + */ +@ChannelHandler.Sharable +public class NettyServerHandler extends SimpleChannelInboundHandler { + private static final Logger LOG = LoggerFactory.getLogger(NettyServerHandler.class); + + private final Map commands = Maps.newHashMap(); + + void registerProcessor(short requestCode, NettyRequestProcessor processor, ExecutorService executor) { + this.commands.put(requestCode, new NettyRequestExecutor(requestCode, processor, executor)); + } + + @Override + protected void channelRead0(ChannelHandlerContext ctx, RemotingCommand command) { + command.setReceiveTime(System.currentTimeMillis()); + processMessageReceived(ctx, command); + } + + private void processMessageReceived(ChannelHandlerContext ctx, RemotingCommand cmd) { + if (cmd != null) { + switch (cmd.getCommandType()) { + case REQUEST_COMMAND: + processRequestCommand(ctx, cmd); + break; + case RESPONSE_COMMAND: + processResponseCommand(ctx, cmd); + break; + default: + break; + } + } + } + + private void processResponseCommand(final ChannelHandlerContext ctx, final RemotingCommand cmd) { + } + + private void processRequestCommand(ChannelHandlerContext ctx, RemotingCommand cmd) { + final NettyRequestExecutor executor = commands.get(cmd.getHeader().getCode()); + if (executor == null) { + cmd.release(); + LOG.error("unknown command code, code: {}", cmd.getHeader().getCode()); + Datagram response = RemotingBuilder.buildEmptyResponseDatagram(CommandCode.UNKNOWN_CODE, cmd.getHeader()); + ctx.writeAndFlush(response); + } else { + executor.execute(ctx, cmd); + } + } +} diff --git a/qmq-server-common/src/main/java/qunar/tc/qmq/service/HeartbeatManager.java b/qmq-server-common/src/main/java/qunar/tc/qmq/service/HeartbeatManager.java new file mode 100644 index 00000000..711acb82 --- /dev/null +++ b/qmq-server-common/src/main/java/qunar/tc/qmq/service/HeartbeatManager.java @@ -0,0 +1,63 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.service; + +import io.netty.util.HashedWheelTimer; +import io.netty.util.Timeout; +import io.netty.util.TimerTask; +import qunar.tc.qmq.common.Disposable; +import qunar.tc.qmq.concurrent.NamedThreadFactory; + +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.TimeUnit; + +/** + * @author yunfeng.yang + * @since 2017/9/19 + */ +public class HeartbeatManager implements Disposable { + private final HashedWheelTimer timer; + + private final ConcurrentMap timeouts; + + public HeartbeatManager() { + this.timeouts = new ConcurrentHashMap<>(); + this.timer = new HashedWheelTimer(new NamedThreadFactory("qmq-heartbeat")); + this.timer.start(); + } + + public void cancel(T key) { + Timeout timeout = timeouts.remove(key); + if (timeout == null) return; + + timeout.cancel(); + } + + public void refreshHeartbeat(T key, TimerTask task, long timeout, TimeUnit unit) { + Timeout context = timer.newTimeout(task, timeout, unit); + final Timeout old = timeouts.put(key, context); + if (old != null && !old.isCancelled() && !old.isExpired()) { + old.cancel(); + } + } + + @Override + public void destroy() { + timer.stop(); + } +} diff --git a/qmq-server/pom.xml b/qmq-server/pom.xml new file mode 100644 index 00000000..b2cf9be7 --- /dev/null +++ b/qmq-server/pom.xml @@ -0,0 +1,72 @@ + + + 4.0.0 + + + qmq + qunar.tc + 4.0.30 + + + qmq-server + jar + + + + ${project.groupId} + qmq-common + + + ${project.groupId} + qmq-server-common + + + ${project.groupId} + qmq-remoting + + + ${project.groupId} + qmq-sync + + + ${project.groupId} + qmq-store + + + ${project.groupId} + qmq-client + + + + org.slf4j + slf4j-api + + + org.slf4j + jcl-over-slf4j + + + org.slf4j + log4j-over-slf4j + + + ch.qos.logback + logback-classic + + + ch.qos.logback + logback-core + + + + com.google.guava + guava + + + + + ROOT + + \ No newline at end of file diff --git a/qmq-server/src/main/java/qunar/tc/qmq/base/ConsumerGroup.java b/qmq-server/src/main/java/qunar/tc/qmq/base/ConsumerGroup.java new file mode 100644 index 00000000..9e977235 --- /dev/null +++ b/qmq-server/src/main/java/qunar/tc/qmq/base/ConsumerGroup.java @@ -0,0 +1,72 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.base; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; + +/** + * @author yunfeng.yang + * @since 2017/7/6 + */ +@JsonDeserialize +public class ConsumerGroup { + private final String subject; + private final String group; + + @JsonCreator + public ConsumerGroup(@JsonProperty("subject") String subject, + @JsonProperty("group") String group) { + this.subject = subject; + this.group = group; + } + + public String getSubject() { + return subject; + } + + public String getGroup() { + return group; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + + ConsumerGroup that = (ConsumerGroup) o; + + if (subject != null ? !subject.equals(that.subject) : that.subject != null) return false; + return group != null ? group.equals(that.group) : that.group == null; + } + + @Override + public int hashCode() { + int result = subject != null ? subject.hashCode() : 0; + result = 31 * result + (group != null ? group.hashCode() : 0); + return result; + } + + @Override + public String toString() { + return "ConsumerGroup{" + + "subject='" + subject + '\'' + + ", group='" + group + '\'' + + '}'; + } +} diff --git a/qmq-server/src/main/java/qunar/tc/qmq/base/ConsumerSequence.java b/qmq-server/src/main/java/qunar/tc/qmq/base/ConsumerSequence.java new file mode 100644 index 00000000..d823a09f --- /dev/null +++ b/qmq-server/src/main/java/qunar/tc/qmq/base/ConsumerSequence.java @@ -0,0 +1,92 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.base; + +import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.locks.Lock; +import java.util.concurrent.locks.ReentrantLock; + +/** + * @author yunfeng.yang + * @since 2017/8/1 + */ +public class ConsumerSequence { + private final AtomicLong pullSequence; + private final AtomicLong ackSequence; + + private final Lock pullLock = new ReentrantLock(); + private final Lock ackLock = new ReentrantLock(); + + public ConsumerSequence(final long pullSequence, final long ackSequence) { + this.pullSequence = new AtomicLong(pullSequence); + this.ackSequence = new AtomicLong(ackSequence); + } + + public long getPullSequence() { + return pullSequence.get(); + } + + public void setPullSequence(long pullSequence) { + this.pullSequence.set(pullSequence); + } + + public long getAckSequence() { + return ackSequence.get(); + } + + public void setAckSequence(long ackSequence) { + this.ackSequence.set(ackSequence); + } + + public void pullLock() { + pullLock.lock(); + } + + public void pullUnlock() { + pullLock.unlock(); + } + + public void ackLock() { + ackLock.lock(); + } + + public void ackUnLock() { + ackLock.unlock(); + } + + public boolean tryLock() { + if (!pullLock.tryLock()) return false; + if (!ackLock.tryLock()) { + pullLock.unlock(); + return false; + } + return true; + } + + public void unlock() { + pullLock.unlock(); + ackLock.unlock(); + } + + @Override + public String toString() { + return "ConsumerSequence{" + + "pullSequence=" + pullSequence + + ", ackSequence=" + ackSequence + + '}'; + } +} diff --git a/qmq-server/src/main/java/qunar/tc/qmq/base/PullMessageResult.java b/qmq-server/src/main/java/qunar/tc/qmq/base/PullMessageResult.java new file mode 100644 index 00000000..21a63e68 --- /dev/null +++ b/qmq-server/src/main/java/qunar/tc/qmq/base/PullMessageResult.java @@ -0,0 +1,69 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.base; + +import qunar.tc.qmq.store.SegmentBuffer; + +import java.util.ArrayList; +import java.util.List; + +/** + * @author yunfeng.yang + * @since 2017/10/30 + */ +public class PullMessageResult { + private final long pullLogOffset; + private final List buffers; + private int bufferTotalSize; + private int messageNum; + + public static final PullMessageResult EMPTY = new PullMessageResult(-1, new ArrayList<>(), 0, 0); + + public static final PullMessageResult FILTER_EMPTY = new PullMessageResult(-1, new ArrayList<>(), 0, 0); + + public PullMessageResult(long pullLogOffset, List buffers, int bufferTotalSize, int messageNum) { + this.pullLogOffset = pullLogOffset; + this.buffers = buffers; + this.bufferTotalSize = bufferTotalSize; + this.messageNum = messageNum; + } + + public long getPullLogOffset() { + return pullLogOffset; + } + + public List getBuffers() { + return buffers; + } + + public int getBufferTotalSize() { + return bufferTotalSize; + } + + public int getMessageNum() { + return messageNum; + } + + @Override + public String toString() { + return "PullMessageResult{" + + "pullLogOffset=" + pullLogOffset + + ", bufferTotalSize=" + bufferTotalSize + + ", messageNum=" + messageNum + + '}'; + } +} diff --git a/qmq-server/src/main/java/qunar/tc/qmq/base/ReceiveResult.java b/qmq-server/src/main/java/qunar/tc/qmq/base/ReceiveResult.java new file mode 100644 index 00000000..8a6f7ba5 --- /dev/null +++ b/qmq-server/src/main/java/qunar/tc/qmq/base/ReceiveResult.java @@ -0,0 +1,61 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.base; + +/** + * @author yunfeng.yang + * @since 2017/8/8 + */ +public class ReceiveResult { + private final String messageId; + private final int code; + private final String remark; + private final long endOffsetOfMessage; + + public ReceiveResult(String messageId, int code, String remark, long endOffsetOfMessage) { + this.messageId = messageId; + this.code = code; + this.remark = remark; + this.endOffsetOfMessage = endOffsetOfMessage; + } + + public String getMessageId() { + return messageId; + } + + public int getCode() { + return code; + } + + public String getRemark() { + return remark; + } + + public long getEndOffsetOfMessage() { + return endOffsetOfMessage; + } + + @Override + public String toString() { + return "ReceiveResult{" + + "messageId='" + messageId + '\'' + + ", code=" + code + + ", remark='" + remark + '\'' + + ", endOffsetOfMessage=" + endOffsetOfMessage + + '}'; + } +} diff --git a/qmq-server/src/main/java/qunar/tc/qmq/base/ReceivingMessage.java b/qmq-server/src/main/java/qunar/tc/qmq/base/ReceivingMessage.java new file mode 100644 index 00000000..916b68c5 --- /dev/null +++ b/qmq-server/src/main/java/qunar/tc/qmq/base/ReceivingMessage.java @@ -0,0 +1,69 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.base; + +import com.google.common.util.concurrent.SettableFuture; + +/** + * @author yunfeng.yang + * @since 2017/8/8 + */ +public class ReceivingMessage { + private final RawMessage message; + private final SettableFuture promise; + + private final long receivedTime; + + public ReceivingMessage(RawMessage message, long receivedTime) { + this.message = message; + this.receivedTime = receivedTime; + this.promise = SettableFuture.create(); + } + + public RawMessage getMessage() { + return message; + } + + public SettableFuture promise() { + return promise; + } + + public long getReceivedTime() { + return receivedTime; + } + + public String getMessageId() { + return message.getHeader().getMessageId(); + } + + public void done(ReceiveResult result) { + promise.set(result); + } + + public String getSubject() { + return message.getHeader().getSubject(); + } + + public boolean isExpired() { + return System.currentTimeMillis() > message.getHeader().getExpireTime(); + } + + public boolean isHigh() { + return message.isHigh(); + } + +} diff --git a/qmq-server/src/main/java/qunar/tc/qmq/base/WritePutActionResult.java b/qmq-server/src/main/java/qunar/tc/qmq/base/WritePutActionResult.java new file mode 100644 index 00000000..7b7274a8 --- /dev/null +++ b/qmq-server/src/main/java/qunar/tc/qmq/base/WritePutActionResult.java @@ -0,0 +1,39 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.base; + +/** + * @author yunfeng.yang + * @since 2017/8/23 + */ +public class WritePutActionResult { + private final boolean status; + private final long pullLogOffset; + + public WritePutActionResult(boolean status, long pullLogOffset) { + this.status = status; + this.pullLogOffset = pullLogOffset; + } + + public boolean isSuccess() { + return status; + } + + public long getPullLogOffset() { + return pullLogOffset; + } +} diff --git a/qmq-server/src/main/java/qunar/tc/qmq/consumer/ConsumerSequenceManager.java b/qmq-server/src/main/java/qunar/tc/qmq/consumer/ConsumerSequenceManager.java new file mode 100644 index 00000000..8f1dfd57 --- /dev/null +++ b/qmq-server/src/main/java/qunar/tc/qmq/consumer/ConsumerSequenceManager.java @@ -0,0 +1,286 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.consumer; + +import com.google.common.collect.Table; +import io.netty.buffer.ByteBuf; +import io.netty.buffer.Unpooled; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import qunar.tc.qmq.base.ConsumerGroup; +import qunar.tc.qmq.base.ConsumerSequence; +import qunar.tc.qmq.base.RawMessage; +import qunar.tc.qmq.base.WritePutActionResult; +import qunar.tc.qmq.monitor.QMon; +import qunar.tc.qmq.processor.AckMessageProcessor; +import qunar.tc.qmq.protocol.QMQSerializer; +import qunar.tc.qmq.store.*; +import qunar.tc.qmq.store.action.ForeverOfflineAction; +import qunar.tc.qmq.store.action.PullAction; +import qunar.tc.qmq.store.action.RangeAckAction; +import qunar.tc.qmq.utils.ObjectUtils; +import qunar.tc.qmq.utils.RetrySubjectUtils; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; + +/** + * @author yunfeng.yang + * @since 2017/8/1 + */ +public class ConsumerSequenceManager { + private static final Logger LOG = LoggerFactory.getLogger(ConsumerSequenceManager.class); + + private static final long ACTION_LOG_ORIGIN_OFFSET = -1L; + + private final Storage storage; + + // subject -> consumer group -> consumer id + private final ConcurrentMap> sequences; + + public ConsumerSequenceManager(final Storage storage) { + this.storage = storage; + this.sequences = new ConcurrentHashMap<>(); + } + + public void init() { + loadFromConsumerGroupProgresses(sequences); + } + + private void loadFromConsumerGroupProgresses(final ConcurrentMap> result) { + final Collection progresses = storage.allConsumerGroupProgresses(); + progresses.forEach(progress -> { + final Map consumers = progress.getConsumers(); + if (consumers == null || consumers.isEmpty()) { + return; + } + + consumers.values().forEach(consumer -> putConsumer(result, consumer)); + }); + } + + private void putConsumer(final ConcurrentMap> result, final ConsumerProgress consumer) { + final String consumerId = consumer.getConsumerId(); + + ConcurrentMap consumerSequences = result.get(consumerId); + if (consumerSequences == null) { + consumerSequences = new ConcurrentHashMap<>(); + result.putIfAbsent(consumerId, consumerSequences); + } + + final ConsumerSequence consumerSequence = new ConsumerSequence(consumer.getPull(), consumer.getAck()); + final ConsumerGroup consumerGroup = new ConsumerGroup(consumer.getSubject(), consumer.getGroup()); + consumerSequences.putIfAbsent(consumerGroup, consumerSequence); + + } + + public WritePutActionResult putPullActions(final String subject, final String group, final String consumerId, final boolean isBroadcast, final GetMessageResult getMessageResult) { + final OffsetRange consumerLogRange = getMessageResult.getConsumerLogRange(); + final ConsumerSequence consumerSequence = getOrCreateConsumerSequence(subject, group, consumerId); + + if (consumerLogRange.getEnd() - consumerLogRange.getBegin() + 1 != getMessageResult.getMessageNum()) { + LOG.debug("consumer offset range error, subject:{}, group:{}, consumerId:{}, isBroadcast:{}, getMessageResult:{}", subject, group, consumerId, isBroadcast, getMessageResult); + QMon.consumerLogOffsetRangeError(subject, group); + } + consumerSequence.pullLock(); + try { + //因为消息堆积等原因,可能会导致历史消息已经被删除了。所以可能得出这种情况:一次拉取100条消息,然后前20条已经删除了,所以不能使用begin,要使用end减去消息条数这种方式 + final long firstConsumerLogSequence = consumerLogRange.getEnd() - getMessageResult.getMessageNum() + 1; + final long lastConsumerLogSequence = consumerLogRange.getEnd(); + + final long firstPullSequence = isBroadcast ? firstConsumerLogSequence : consumerSequence.getPullSequence() + 1; + final long lastPullSequence = isBroadcast ? lastConsumerLogSequence : consumerSequence.getPullSequence() + getMessageResult.getMessageNum(); + + final Action action = new PullAction(subject, group, consumerId, + System.currentTimeMillis(), isBroadcast, + firstPullSequence, lastPullSequence, + firstConsumerLogSequence, lastConsumerLogSequence); + + if (!putAction(action)) { + return new WritePutActionResult(false, -1); + } + consumerSequence.setPullSequence(lastPullSequence); + return new WritePutActionResult(true, firstPullSequence); + } catch (Exception e) { + LOG.error("write action log failed, subject: {}, group: {}, consumerId: {}", subject, group, consumerId, e); + return new WritePutActionResult(false, -1); + } finally { + consumerSequence.pullUnlock(); + } + } + + public boolean putAckActions(AckMessageProcessor.AckEntry ackEntry) { + final String consumerId = ackEntry.getConsumerId(); + final String subject = ackEntry.getSubject(); + final String group = ackEntry.getGroup(); + final long lastPullSequence = ackEntry.getLastPullLogOffset(); + long firstPullSequence = ackEntry.getFirstPullLogOffset(); + + final ConsumerSequence consumerSequence = getOrCreateConsumerSequence(subject, group, consumerId); + + consumerSequence.ackLock(); + final long confirmedAckSequence = consumerSequence.getAckSequence(); + try { + if (lastPullSequence <= confirmedAckSequence) { + LOG.warn("receive duplicate ack, ackEntry:{}, consumerSequence:{} ", ackEntry, consumerSequence); + QMon.consumerDuplicateAckCountInc(subject, group, (int) (confirmedAckSequence - lastPullSequence)); + return true; + } + final long lostAckCount = firstPullSequence - confirmedAckSequence; + if (lostAckCount <= 0) { + LOG.warn("receive some duplicate ack, ackEntry:{}, consumerSequence:{}", ackEntry, consumerSequence); + firstPullSequence = confirmedAckSequence + 1; + QMon.consumerDuplicateAckCountInc(subject, group, (int) (confirmedAckSequence - firstPullSequence)); + } else if (lostAckCount > 1) { + final long firstNotAckedPullSequence = confirmedAckSequence + 1; + final long lastLostPullSequence = firstPullSequence - 1; + //如果是广播的话,put need retry也是没有意义的 + if (!ackEntry.isBroadcast()) { + LOG.error("lost ack count, ackEntry:{}, consumerSequence:{}", ackEntry, consumerSequence); + putNeedRetryMessages(subject, group, consumerId, firstNotAckedPullSequence, lastLostPullSequence); + } + firstPullSequence = firstNotAckedPullSequence; + QMon.consumerLostAckCountInc(subject, group, (int) lostAckCount); + } + + final Action rangeAckAction = new RangeAckAction(subject, group, consumerId, System.currentTimeMillis(), firstPullSequence, lastPullSequence); + if (!putAction(rangeAckAction)) + return false; + + consumerSequence.setAckSequence(lastPullSequence); + return true; + } catch (Exception e) { + QMon.putAckActionsErrorCountInc(ackEntry.getSubject(), ackEntry.getGroup()); + LOG.error("put ack actions error, ackEntry:{}, consumerSequence:{}", ackEntry, consumerSequence, e); + return false; + } finally { + consumerSequence.ackUnLock(); + } + } + + boolean putForeverOfflineAction(final String subject, final String group, final String consumerId) { + final ForeverOfflineAction action = new ForeverOfflineAction(subject, group, consumerId, System.currentTimeMillis()); + return putAction(action); + } + + public boolean putAction(final Action action) { + final PutMessageResult putMessageResult = storage.putAction(action); + if (putMessageResult.getStatus() == PutMessageStatus.SUCCESS) { + return true; + } + + LOG.error("put action fail, action:{}", action); + QMon.putActionFailedCountInc(action.subject(), action.group()); + return false; + } + + void putNeedRetryMessages(String subject, String group, String consumerId, long firstNotAckedOffset, long lastPullLogOffset) { + if (noPullLog(subject, group, consumerId)) return; + + // get error msg + final List needRetryMessages = getNeedRetryMessages(subject, group, consumerId, firstNotAckedOffset, lastPullLogOffset); + // put error msg + putNeedRetryMessages(subject, group, consumerId, needRetryMessages); + } + + private boolean noPullLog(String subject, String group, String consumerId) { + Table pullLogs = storage.allPullLogs(); + Map subscribers = pullLogs.row(consumerId); + if (subscribers == null || subscribers.isEmpty()) return true; + return subscribers.get(GroupAndSubject.groupAndSubject(subject, group)) == null; + } + + void remove(String subject, String group, String consumerId) { + final ConcurrentMap consumers = sequences.get(consumerId); + if (consumers == null) return; + + consumers.remove(new ConsumerGroup(subject, group)); + if (consumers.isEmpty()) { + sequences.remove(consumerId); + } + } + + private List getNeedRetryMessages(String subject, String group, String consumerId, long firstNotAckedSequence, long lastPullSequence) { + final int actualNum = (int) (lastPullSequence - firstNotAckedSequence + 1); + final List needRetryMessages = new ArrayList<>(actualNum); + for (long sequence = firstNotAckedSequence; sequence <= lastPullSequence; sequence++) { + final long consumerLogSequence = storage.getMessageSequenceByPullLog(subject, group, consumerId, sequence); + if (consumerLogSequence < 0) { + LOG.warn("find no consumer log offset for this pull log, subject:{}, group:{}, consumerId:{}, sequence:{}, consumerLogSequence:{}", subject, group, consumerId, sequence, consumerLogSequence); + continue; + } + + final GetMessageResult getMessageResult = storage.getMessage(subject, consumerLogSequence); + if (getMessageResult.getStatus() == GetMessageStatus.SUCCESS) { + final List segmentBuffers = getMessageResult.getSegmentBuffers(); + needRetryMessages.addAll(segmentBuffers); + } + } + return needRetryMessages; + } + + private void putNeedRetryMessages(String subject, String group, String consumerId, List needRetryMessages) { + try { + for (SegmentBuffer buffer : needRetryMessages) { + final ByteBuf message = Unpooled.wrappedBuffer(buffer.getBuffer()); + final RawMessage rawMessage = QMQSerializer.deserializeRawMessage(message); + if (!RetrySubjectUtils.isRetrySubject(subject)) { + final String retrySubject = RetrySubjectUtils.buildRetrySubject(subject, group); + rawMessage.setSubject(retrySubject); + } + + final PutMessageResult putMessageResult = storage.appendMessage(rawMessage); + if (putMessageResult.getStatus() != PutMessageStatus.SUCCESS) { + LOG.error("put message error, consumer:{} {} {}, status:{}", subject, group, consumerId, putMessageResult.getStatus()); + throw new RuntimeException("put retry message error"); + } + } + } finally { + needRetryMessages.forEach(SegmentBuffer::release); + } + + QMon.putNeedRetryMessagesCountInc(subject, group, needRetryMessages.size()); + } + + public ConsumerSequence getConsumerSequence(String subject, String group, String consumerId) { + final ConcurrentMap consumerSequences = this.sequences.get(consumerId); + if (consumerSequences == null) { + return null; + } + return consumerSequences.get(new ConsumerGroup(subject, group)); + } + + public ConsumerSequence getOrCreateConsumerSequence(String subject, String group, String consumerId) { + ConcurrentMap consumerSequences = this.sequences.get(consumerId); + if (consumerSequences == null) { + final ConcurrentMap newConsumerSequences = new ConcurrentHashMap<>(); + consumerSequences = ObjectUtils.defaultIfNull(sequences.putIfAbsent(consumerId, newConsumerSequences), newConsumerSequences); + } + + final ConsumerGroup consumerGroup = new ConsumerGroup(subject, group); + ConsumerSequence consumerSequence = consumerSequences.get(consumerGroup); + if (consumerSequence == null) { + final ConsumerSequence newConsumerSequence = new ConsumerSequence(ACTION_LOG_ORIGIN_OFFSET, ACTION_LOG_ORIGIN_OFFSET); + consumerSequence = ObjectUtils.defaultIfNull(consumerSequences.putIfAbsent(consumerGroup, newConsumerSequence), newConsumerSequence); + } + return consumerSequence; + } +} diff --git a/qmq-server/src/main/java/qunar/tc/qmq/consumer/OfflineActionHandler.java b/qmq-server/src/main/java/qunar/tc/qmq/consumer/OfflineActionHandler.java new file mode 100644 index 00000000..b0078ff8 --- /dev/null +++ b/qmq-server/src/main/java/qunar/tc/qmq/consumer/OfflineActionHandler.java @@ -0,0 +1,59 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.consumer; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import qunar.tc.qmq.store.Action; +import qunar.tc.qmq.store.Storage; +import qunar.tc.qmq.store.action.ActionEvent; +import qunar.tc.qmq.store.event.FixedExecOrderEventBus; + +/** + * @author keli.wang + * @since 2018/8/21 + */ +public class OfflineActionHandler implements FixedExecOrderEventBus.Listener { + private static final Logger LOG = LoggerFactory.getLogger(OfflineActionHandler.class); + + private final Storage storage; + + public OfflineActionHandler(Storage storage) { + this.storage = storage; + } + + @Override + public void onEvent(ActionEvent event) { + switch (event.getAction().type()) { + case FOREVER_OFFLINE: + foreverOffline(event.getAction()); + break; + default: + break; + + } + } + + private void foreverOffline(final Action action) { + final String subject = action.subject(); + final String group = action.group(); + final String consumerId = action.consumerId(); + + LOG.info("execute offline task, will remove pull log and checkpoint entry for {}/{}/{}", subject, group, consumerId); + storage.destroyPullLog(subject, group, consumerId); + } +} diff --git a/qmq-server/src/main/java/qunar/tc/qmq/consumer/OfflineTask.java b/qmq-server/src/main/java/qunar/tc/qmq/consumer/OfflineTask.java new file mode 100644 index 00000000..671b89d1 --- /dev/null +++ b/qmq-server/src/main/java/qunar/tc/qmq/consumer/OfflineTask.java @@ -0,0 +1,88 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.consumer; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import qunar.tc.qmq.base.ConsumerSequence; +import qunar.tc.qmq.monitor.QMon; + +/** + * Created by zhaohui.yu + * 7/31/18 + */ +class OfflineTask { + private static final Logger LOG = LoggerFactory.getLogger(OfflineTask.class); + + private final ConsumerSequenceManager consumerSequenceManager; + private final Subscriber subscriber; + + private volatile boolean cancel = false; + + OfflineTask(ConsumerSequenceManager consumerSequenceManager, Subscriber subscriber) { + this.consumerSequenceManager = consumerSequenceManager; + this.subscriber = subscriber; + } + + void run() { + if (cancel) return; + + LOG.info("run offline task for {}/{}/{}.", subscriber.getSubject(), subscriber.getGroup(), subscriber.getConsumerId()); + QMon.offlineTaskExecuteCountInc(subscriber.getSubject(), subscriber.getGroup()); + + final ConsumerSequence consumerSequence = consumerSequenceManager.getOrCreateConsumerSequence(subscriber.getSubject(), subscriber.getGroup(), subscriber.getConsumerId()); + if (!consumerSequence.tryLock()) return; + try { + if (cancel) return; + + if (isProcessedComplete(consumerSequence)) { + if (unSubscribe()) { + LOG.info("offline task destroyed subscriber for {}/{}/{}", subscriber.getSubject(), subscriber.getGroup(), subscriber.getConsumerId()); + } + } else { + LOG.info("offline task skip destroy subscriber for {}/{}/{}", subscriber.getSubject(), subscriber.getGroup(), subscriber.getConsumerId()); + } + } finally { + consumerSequence.unlock(); + } + } + + private boolean isProcessedComplete(final ConsumerSequence consumerSequence) { + final long lastAckedSequence = consumerSequence.getAckSequence(); + final long lastPulledSequence = consumerSequence.getPullSequence(); + return lastPulledSequence <= lastAckedSequence; + } + + private boolean unSubscribe() { + if (cancel) return false; + final boolean success = consumerSequenceManager.putForeverOfflineAction(subscriber.getSubject(), subscriber.getGroup(), subscriber.getConsumerId()); + if (!success) { + return false; + } + consumerSequenceManager.remove(subscriber.getSubject(), subscriber.getGroup(), subscriber.getConsumerId()); + subscriber.destroy(); + return true; + } + + void cancel() { + cancel = true; + } + + void reset() { + cancel = false; + } +} diff --git a/qmq-server/src/main/java/qunar/tc/qmq/consumer/RetryTask.java b/qmq-server/src/main/java/qunar/tc/qmq/consumer/RetryTask.java new file mode 100644 index 00000000..5449d0f3 --- /dev/null +++ b/qmq-server/src/main/java/qunar/tc/qmq/consumer/RetryTask.java @@ -0,0 +1,139 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.consumer; + +import com.google.common.util.concurrent.RateLimiter; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import qunar.tc.qmq.base.ConsumerSequence; +import qunar.tc.qmq.configuration.DynamicConfig; +import qunar.tc.qmq.monitor.QMon; +import qunar.tc.qmq.store.Action; +import qunar.tc.qmq.store.action.RangeAckAction; + +/** + * Created by zhaohui.yu + * 7/30/18 + */ +class RetryTask { + private static final Logger LOG = LoggerFactory.getLogger(RetryTask.class); + + private final DynamicConfig config; + private final ConsumerSequenceManager consumerSequenceManager; + private final Subscriber subscriber; + private final RateLimiter limiter; + + private volatile boolean cancel; + + RetryTask(DynamicConfig config, ConsumerSequenceManager consumerSequenceManager, Subscriber subscriber) { + this.config = config; + this.consumerSequenceManager = consumerSequenceManager; + this.subscriber = subscriber; + this.limiter = RateLimiter.create(50); + this.config.addListener(conf -> updateLimitRate(conf, "put_need_retry_message.limiter")); + } + + private void updateLimitRate(DynamicConfig conf, final String key) { + if (!conf.exist(key)) { + return; + } + + try { + final double limit = conf.getDouble(key); + limiter.setRate(limit); + } catch (Exception e) { + LOG.debug("update limiter rate failed", e); + } + } + + void run() { + if (cancel) return; + + final ConsumerSequence consumerSequence = consumerSequenceManager.getConsumerSequence(subscriber.getSubject(), subscriber.getGroup(), subscriber.getConsumerId()); + if (consumerSequence == null) { + return; + } + + QMon.retryTaskExecuteCountInc(subscriber.getSubject(), subscriber.getGroup()); + + if (processSkipRetry(consumerSequence)) return; + + while (true) { + limiter.acquire(); + + if (!consumerSequence.tryLock()) return; + try { + if (cancel) return; + + final long firstNotAckedSequence = consumerSequence.getAckSequence() + 1; + final long lastPulledSequence = consumerSequence.getPullSequence(); + if (lastPulledSequence < firstNotAckedSequence) return; + + subscriber.renew(); + + if (isDryRun()) { + LOG.info("dry run retry task, subject: {}, group: {}, consumerId: {}, firstNotAckedSequence: {}, lastPulledSequence: {}", + subscriber.getSubject(), subscriber.getGroup(), subscriber.getConsumerId(), firstNotAckedSequence, lastPulledSequence); + return; + } + + LOG.info("put need retry message in retry task, subject: {}, group: {}, consumerId: {}, ack offset: {}, pull offset: {}", + subscriber.getSubject(), subscriber.getGroup(), subscriber.getConsumerId(), firstNotAckedSequence, lastPulledSequence); + consumerSequenceManager.putNeedRetryMessages(subscriber.getSubject(), subscriber.getGroup(), subscriber.getConsumerId(), firstNotAckedSequence, firstNotAckedSequence); + + // put ack action + final Action action = new RangeAckAction(subscriber.getSubject(), subscriber.getGroup(), subscriber.getConsumerId(), System.currentTimeMillis(), firstNotAckedSequence, firstNotAckedSequence); + if (consumerSequenceManager.putAction(action)) { + consumerSequence.setAckSequence(firstNotAckedSequence); + QMon.consumerAckTimeoutErrorCountInc(subscriber.getConsumerId(), 1); + } + } finally { + consumerSequence.unlock(); + } + } + } + + private boolean processSkipRetry(ConsumerSequence consumerSequence) { + if (!consumerSequence.tryLock()) return true; + try { + final long firstNotAckedSequence = consumerSequence.getAckSequence() + 1; + final long lastPulledSequence = consumerSequence.getPullSequence(); + if (lastPulledSequence < firstNotAckedSequence) return true; + + // put ack action + final Action action = new RangeAckAction(subscriber.getSubject(), subscriber.getGroup(), subscriber.getConsumerId(), System.currentTimeMillis(), firstNotAckedSequence, lastPulledSequence); + if (consumerSequenceManager.putAction(action)) { + consumerSequence.setAckSequence(lastPulledSequence); + } + } finally { + consumerSequence.unlock(); + } + return true; + } + + private boolean isDryRun() { + return config.getBoolean("ConsumerStatusChecker.RetryTask.DryRun", false); + } + + void cancel() { + cancel = true; + } + + void reset() { + cancel = false; + } +} diff --git a/qmq-server/src/main/java/qunar/tc/qmq/consumer/Subscriber.java b/qmq-server/src/main/java/qunar/tc/qmq/consumer/Subscriber.java new file mode 100644 index 00000000..0f73cbb9 --- /dev/null +++ b/qmq-server/src/main/java/qunar/tc/qmq/consumer/Subscriber.java @@ -0,0 +1,139 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.consumer; + +import qunar.tc.qmq.store.GroupAndSubject; + +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; + +/** + * Created by zhaohui.yu + * 7/30/18 + */ +class Subscriber { + //3分钟无心跳,则认为暂时离线 + private static final long OFFLINE_LEASE_MILLIS = TimeUnit.MINUTES.toMillis(3); + + //2天都没有心跳则认为该consumer永久离线 + private static final long FOREVER_LEASE_MILLIS = TimeUnit.MINUTES.toMillis(10); + + private final SubscriberStatusChecker checker; + private final String name; + + private final String subject; + private final String group; + private final String consumerId; + + private RetryTask retryTask; + private OfflineTask offlineTask; + private volatile long lastUpdate; + + private final AtomicBoolean processed = new AtomicBoolean(false); + + Subscriber(SubscriberStatusChecker checker, String name, String consumerId) { + this.checker = checker; + this.name = name; + + final GroupAndSubject groupAndSubject = GroupAndSubject.parse(name); + this.group = groupAndSubject.getGroup(); + this.subject = groupAndSubject.getSubject(); + this.consumerId = consumerId; + + this.lastUpdate = System.currentTimeMillis(); + } + + public String name() { + return name; + } + + public String getSubject() { + return subject; + } + + public String getGroup() { + return group; + } + + public String getConsumerId() { + return consumerId; + } + + void setRetryTask(RetryTask retryTask) { + this.retryTask = retryTask; + } + + void setOfflineTask(OfflineTask offlineTask) { + this.offlineTask = offlineTask; + } + + void checkStatus() { + try { + Status status = status(); + if (status == Status.OFFLINE) { + if (processed.compareAndSet(false, true)) { + retryTask.run(); + } + } + if (status == Status.FOREVER) { + if (processed.compareAndSet(false, true)) { + offlineTask.run(); + } + } + } finally { + processed.set(false); + } + } + + Status status() { + long now = System.currentTimeMillis(); + long interval = now - lastUpdate; + if (interval >= FOREVER_LEASE_MILLIS) { + return Status.FOREVER; + } + if (interval >= OFFLINE_LEASE_MILLIS) { + return Status.OFFLINE; + } + + return Status.ONLINE; + } + + void heartbeat() { + renew(); + retryTask.cancel(); + offlineTask.cancel(); + } + + void renew() { + lastUpdate = System.currentTimeMillis(); + } + + void reset() { + retryTask.reset(); + offlineTask.reset(); + } + + public void destroy() { + checker.destroy(this); + } + + public enum Status { + ONLINE, + OFFLINE, + FOREVER + } +} diff --git a/qmq-server/src/main/java/qunar/tc/qmq/consumer/SubscriberStatusChecker.java b/qmq-server/src/main/java/qunar/tc/qmq/consumer/SubscriberStatusChecker.java new file mode 100644 index 00000000..9ad2d35f --- /dev/null +++ b/qmq-server/src/main/java/qunar/tc/qmq/consumer/SubscriberStatusChecker.java @@ -0,0 +1,233 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.consumer; + +import com.google.common.collect.Table; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import qunar.tc.qmq.common.Disposable; +import qunar.tc.qmq.concurrent.ActorSystem; +import qunar.tc.qmq.concurrent.NamedThreadFactory; +import qunar.tc.qmq.configuration.DynamicConfig; +import qunar.tc.qmq.store.ConsumerGroupProgress; +import qunar.tc.qmq.store.GroupAndSubject; +import qunar.tc.qmq.store.PullLog; +import qunar.tc.qmq.store.Storage; +import qunar.tc.qmq.utils.RetrySubjectUtils; + +import java.util.Collection; +import java.util.Map; +import java.util.concurrent.*; + +/** + * Created by zhaohui.yu + * 7/30/18 + */ +public class SubscriberStatusChecker implements ActorSystem.Processor, Runnable, Disposable { + private static final Logger LOG = LoggerFactory.getLogger(SubscriberStatusChecker.class); + + private final DynamicConfig config; + private final Storage storage; + private final ConsumerSequenceManager consumerSequenceManager; + private final ActorSystem actorSystem; + + private final ConcurrentMap> subscribers = new ConcurrentHashMap<>(); + + private volatile boolean online = false; + + private ScheduledExecutorService executor; + + public SubscriberStatusChecker(DynamicConfig config, Storage storage, ConsumerSequenceManager consumerSequenceManager) { + this.config = config; + this.storage = storage; + this.consumerSequenceManager = consumerSequenceManager; + this.actorSystem = new ActorSystem("consumer-consumers", 4, false); + } + + public void init() { + cleanPullLogAndCheckpoint(); + initSubscribers(); + } + + private void cleanPullLogAndCheckpoint() { + final Table pullLogs = storage.allPullLogs(); + if (pullLogs == null || pullLogs.size() == 0) return; + + // delete all pull log without max pulled message sequence + for (final String groupAndSubject : pullLogs.columnKeySet()) { + final GroupAndSubject gs = GroupAndSubject.parse(groupAndSubject); + final long maxPulledMessageSequence = storage.getMaxPulledMessageSequence(gs.getSubject(), gs.getGroup()); + if (maxPulledMessageSequence == -1) { + for (final Map.Entry entry : pullLogs.column(groupAndSubject).entrySet()) { + final String consumerId = entry.getKey(); + LOG.info("remove pull log. subject: {}, group: {}, consumerId: {}", gs.getSubject(), gs.getGroup(), consumerId); + storage.destroyPullLog(gs.getSubject(), gs.getGroup(), consumerId); + } + } + } + } + + private void initSubscribers() { + final Collection progresses = storage.allConsumerGroupProgresses(); + progresses.forEach(progress -> { + if (progress.isBroadcast()) { + return; + } + + progress.getConsumers().values().forEach(consumer -> { + final String groupAndSubject = GroupAndSubject.groupAndSubject(consumer.getSubject(), consumer.getGroup()); + addSubscriber(groupAndSubject, consumer.getConsumerId()); + }); + }); + } + + public void start() { + executor = Executors.newSingleThreadScheduledExecutor(new NamedThreadFactory("consumer-checker")); + executor.scheduleWithFixedDelay(this, 5, 3, TimeUnit.MINUTES); + } + + public void brokerStatusChanged(final Boolean online) { + LOG.info("broker online status changed from {} to {}", this.online, online); + this.online = online; + } + + @Override + public void run() { + try { + check(); + } catch (Throwable e) { + LOG.error("consumer status checker task failed.", e); + } + } + + private void check() { + for (final ConcurrentMap m : subscribers.values()) { + for (final Subscriber subscriber : m.values()) { + if (needSkipCheck()) { + subscriber.renew(); + } else { + if (subscriber.status() == Subscriber.Status.ONLINE) continue; + + subscriber.reset(); + actorSystem.dispatch("status-checker-" + subscriber.getGroup(), subscriber, this); + } + } + } + } + + private boolean needSkipCheck() { + if (online) { + return false; + } + + return config.getBoolean("ConsumerStatusChecker.SkipCheckDuringOffline", true); + } + + @Override + public boolean process(Subscriber subscriber, ActorSystem.Actor self) { + subscriber.checkStatus(); + return true; + } + + public Subscriber getSubscriber(String subject, String group, String consumerId) { + final String groupAndSubject = GroupAndSubject.groupAndSubject(subject, group); + final ConcurrentMap m = subscribers.get(groupAndSubject); + if (m == null) { + return null; + } + + return m.get(consumerId); + } + + public void addSubscriber(String subject, String group, String consumerId) { + final String groupAndSubject = GroupAndSubject.groupAndSubject(subject, group); + addSubscriber(groupAndSubject, consumerId); + } + + private void addSubscriber(String groupAndSubject, String consumerId) { + ConcurrentMap m = subscribers.get(groupAndSubject); + if (m == null) { + m = new ConcurrentHashMap<>(); + final ConcurrentMap old = subscribers.putIfAbsent(groupAndSubject, m); + if (old != null) { + m = old; + } + } + + if (!m.containsKey(consumerId)) { + m.putIfAbsent(consumerId, createSubscriber(groupAndSubject, consumerId)); + } + } + + private Subscriber createSubscriber(String groupAndSubject, String consumerId) { + final Subscriber subscriber = new Subscriber(this, groupAndSubject, consumerId); + final RetryTask retryTask = new RetryTask(config, consumerSequenceManager, subscriber); + final OfflineTask offlineTask = new OfflineTask(consumerSequenceManager, subscriber); + subscriber.setRetryTask(retryTask); + subscriber.setOfflineTask(offlineTask); + return subscriber; + } + + public void heartbeat(String consumerId, String subject, String group) { + final String realSubject = RetrySubjectUtils.getRealSubject(subject); + final String retrySubject = RetrySubjectUtils.buildRetrySubject(realSubject, group); + + refreshSubscriber(realSubject, group, consumerId); + refreshSubscriber(retrySubject, group, consumerId); + } + + private void refreshSubscriber(final String subject, final String group, final String consumerId) { + final Subscriber subscriber = getSubscriber(subject, group, consumerId); + if (subscriber != null) { + subscriber.heartbeat(); + } + } + + // TODO(keli.wang): cannot remove maxPulledMessageSequence here for now, because slave may cannot replay this correctly + public void destroy(Subscriber subscriber) { + final String groupAndSubject = subscriber.name(); + + if (!subscribers.containsKey(groupAndSubject)) { + return; + } + + final ConcurrentMap m = subscribers.get(groupAndSubject); + if (m == null) { + return; + } + m.remove(subscriber.getConsumerId()); + + if (m.isEmpty()) { + handleGroupOffline(subscriber); + } + } + + private void handleGroupOffline(final Subscriber lastSubscriber) { + try { + storage.disableLagMonitor(lastSubscriber.getSubject(), lastSubscriber.getGroup()); + } catch (Throwable e) { + LOG.error("disable monitor error", e); + } + // TODO(keli.wang): how to detect group offline if master and slave's subscriber list is different + } + + @Override + public void destroy() { + if (executor == null) return; + executor.shutdown(); + } +} diff --git a/qmq-server/src/main/java/qunar/tc/qmq/container/Bootstrap.java b/qmq-server/src/main/java/qunar/tc/qmq/container/Bootstrap.java new file mode 100644 index 00000000..1c72208b --- /dev/null +++ b/qmq-server/src/main/java/qunar/tc/qmq/container/Bootstrap.java @@ -0,0 +1,27 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.container; + +import qunar.tc.qmq.configuration.DynamicConfigLoader; +import qunar.tc.qmq.startup.ServerWrapper; + +public class Bootstrap { + public static void main(String[] args) { + ServerWrapper wrapper = new ServerWrapper(DynamicConfigLoader.load("broker.properties")); + wrapper.start(); + } +} diff --git a/qmq-server/src/main/java/qunar/tc/qmq/lag/ConsumerLagService.java b/qmq-server/src/main/java/qunar/tc/qmq/lag/ConsumerLagService.java new file mode 100644 index 00000000..435ac6cc --- /dev/null +++ b/qmq-server/src/main/java/qunar/tc/qmq/lag/ConsumerLagService.java @@ -0,0 +1,73 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.lag; + +import qunar.tc.qmq.base.ConsumerLag; +import qunar.tc.qmq.store.ConsumeQueue; +import qunar.tc.qmq.store.ConsumerGroupProgress; +import qunar.tc.qmq.store.ConsumerProgress; +import qunar.tc.qmq.store.Storage; + +import java.util.HashMap; +import java.util.Map; + +/** + * @author keli.wang + * @since 2018/7/31 + */ +public class ConsumerLagService { + private final Storage storage; + + public ConsumerLagService(final Storage storage) { + this.storage = storage; + } + + public Map getSubjectConsumerLag(final String subject) { + final Map consumeQueues = storage.locateSubjectConsumeQueues(subject); + final Map lags = new HashMap<>(); + for (final Map.Entry entry : consumeQueues.entrySet()) { + final String group = entry.getKey(); + final ConsumeQueue consumeQueue = entry.getValue(); + final ConsumerGroupProgress progress = storage.getConsumerGroupProgress(subject, group); + + final long pullLag = consumeQueue.getQueueCount(); + final long ackLag = computeAckLag(progress); + lags.put(group, new ConsumerLag(pullLag, ackLag)); + } + return lags; + } + + + private long computeAckLag(final ConsumerGroupProgress progress) { + if (progress == null) { + return 0; + } + + final Map consumers = progress.getConsumers(); + if (consumers == null || consumers.isEmpty()) { + return progress.getPull(); + } + + long totalAckLag = 0; + for (final ConsumerProgress consumer : consumers.values()) { + // TODO(keli.wang): check if pull - ack is the right lag + final long ackLag = consumer.getPull() - consumer.getAck(); + totalAckLag += ackLag; + } + return totalAckLag; + } +} diff --git a/qmq-server/src/main/java/qunar/tc/qmq/processor/AbstractRequestProcessor.java b/qmq-server/src/main/java/qunar/tc/qmq/processor/AbstractRequestProcessor.java new file mode 100644 index 00000000..59358cb2 --- /dev/null +++ b/qmq-server/src/main/java/qunar/tc/qmq/processor/AbstractRequestProcessor.java @@ -0,0 +1,32 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.processor; + +import qunar.tc.qmq.meta.BrokerRole; +import qunar.tc.qmq.configuration.BrokerConfig; +import qunar.tc.qmq.netty.NettyRequestProcessor; + +/** + * Created by zhaohui.yu + * 6/19/18 + */ +public abstract class AbstractRequestProcessor implements NettyRequestProcessor { + @Override + public boolean rejectRequest() { + return BrokerConfig.getBrokerRole() == BrokerRole.SLAVE; + } +} diff --git a/qmq-server/src/main/java/qunar/tc/qmq/processor/AckMessageProcessor.java b/qmq-server/src/main/java/qunar/tc/qmq/processor/AckMessageProcessor.java new file mode 100644 index 00000000..d39542aa --- /dev/null +++ b/qmq-server/src/main/java/qunar/tc/qmq/processor/AckMessageProcessor.java @@ -0,0 +1,184 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.processor; + +import com.google.common.base.Strings; +import io.netty.buffer.ByteBuf; +import io.netty.channel.ChannelHandlerContext; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import qunar.tc.qmq.concurrent.ActorSystem; +import qunar.tc.qmq.consumer.ConsumerSequenceManager; +import qunar.tc.qmq.consumer.SubscriberStatusChecker; +import qunar.tc.qmq.monitor.QMon; +import qunar.tc.qmq.protocol.CommandCode; +import qunar.tc.qmq.protocol.Datagram; +import qunar.tc.qmq.protocol.RemotingCommand; +import qunar.tc.qmq.protocol.RemotingHeader; +import qunar.tc.qmq.protocol.consumer.AckRequest; +import qunar.tc.qmq.stats.BrokerStats; +import qunar.tc.qmq.util.RemotingBuilder; +import qunar.tc.qmq.utils.PayloadHolderUtils; + +import java.util.concurrent.CompletableFuture; + +/** + * @author yunfeng.yang + * @since 2017/7/27 + */ +public class AckMessageProcessor extends AbstractRequestProcessor { + private static final Logger LOG = LoggerFactory.getLogger(AckMessageProcessor.class); + + private final AckMessageWorker ackMessageWorker; + private final SubscriberStatusChecker subscriberStatusChecker; + + public AckMessageProcessor(final ActorSystem actorSystem, final ConsumerSequenceManager consumerSequenceManager, final SubscriberStatusChecker subscriberStatusChecker) { + this.ackMessageWorker = new AckMessageWorker(actorSystem, consumerSequenceManager); + this.subscriberStatusChecker = subscriberStatusChecker; + } + + @Override + public CompletableFuture processRequest(ChannelHandlerContext ctx, RemotingCommand command) { + final AckRequest ackRequest = deserializeAckRequest(command); + + BrokerStats.getInstance().getLastMinuteAckRequestCount().add(1); + if (isInvalidRequest(ackRequest)) { + final Datagram datagram = RemotingBuilder.buildEmptyResponseDatagram(CommandCode.BROKER_ERROR, command.getHeader()); + return CompletableFuture.completedFuture(datagram); + } + + QMon.ackRequestCountInc(ackRequest.getSubject(), ackRequest.getGroup()); + subscriberStatusChecker.heartbeat(ackRequest.getConsumerId(), ackRequest.getSubject(), ackRequest.getGroup()); + + if (isHeartbeatAck(ackRequest)) { + final Datagram datagram = RemotingBuilder.buildEmptyResponseDatagram(CommandCode.SUCCESS, command.getHeader()); + return CompletableFuture.completedFuture(datagram); + } + + monitorAckSize(ackRequest); + ackMessageWorker.ack(new AckEntry(ackRequest, ctx, command.getHeader())); + return null; + } + + private AckRequest deserializeAckRequest(RemotingCommand command) { + ByteBuf input = command.getBody(); + AckRequest request = new AckRequest(); + request.setSubject(PayloadHolderUtils.readString(input)); + request.setGroup(PayloadHolderUtils.readString(input)); + request.setConsumerId(PayloadHolderUtils.readString(input)); + request.setPullOffsetBegin(input.readLong()); + request.setPullOffsetLast(input.readLong()); + if (command.getHeader().getVersion() >= RemotingHeader.VERSION_8) { + request.setBroadcast(input.readByte()); + } + return request; + } + + private boolean isInvalidRequest(AckRequest ackRequest) { + if (Strings.isNullOrEmpty(ackRequest.getSubject()) + || Strings.isNullOrEmpty(ackRequest.getGroup()) + || Strings.isNullOrEmpty(ackRequest.getConsumerId())) { + LOG.warn("receive error param ack request: {}", ackRequest); + return true; + } + + return false; + } + + private boolean isHeartbeatAck(AckRequest ackRequest) { + return ackRequest.getPullOffsetBegin() < 0; + } + + private void monitorAckSize(AckRequest ackRequest) { + final int ackSize = (int) (ackRequest.getPullOffsetLast() - ackRequest.getPullOffsetBegin() + 1); + QMon.consumerAckCountInc(ackRequest.getSubject(), ackRequest.getGroup(), ackSize); + } + + public static class AckEntry { + private final String subject; + private final String group; + private final String consumerId; + private final long firstPullLogOffset; + private final long lastPullLogOffset; + private final ChannelHandlerContext ctx; + private final RemotingHeader requestHeader; + private final long ackBegin; + private final byte isBroadcast; + + AckEntry(AckRequest ackRequest, ChannelHandlerContext ctx, RemotingHeader requestHeader) { + this.subject = ackRequest.getSubject(); + this.group = ackRequest.getGroup(); + this.consumerId = ackRequest.getConsumerId(); + this.firstPullLogOffset = ackRequest.getPullOffsetBegin(); + this.lastPullLogOffset = ackRequest.getPullOffsetLast(); + this.isBroadcast = ackRequest.isBroadcast(); + + this.ctx = ctx; + this.requestHeader = requestHeader; + this.ackBegin = System.currentTimeMillis(); + } + + public long getFirstPullLogOffset() { + return firstPullLogOffset; + } + + public long getLastPullLogOffset() { + return lastPullLogOffset; + } + + public String getSubject() { + return subject; + } + + public String getGroup() { + return group; + } + + public String getConsumerId() { + return consumerId; + } + + ChannelHandlerContext getCtx() { + return ctx; + } + + RemotingHeader getRequestHeader() { + return requestHeader; + } + + long getAckBegin() { + return ackBegin; + } + + public boolean isBroadcast() { + return isBroadcast == 1; + } + + @Override + public String toString() { + return "AckEntry{" + + "subject='" + subject + '\'' + + ", group='" + group + '\'' + + ", consumerId='" + consumerId + '\'' + + ", firstPullLogOffset=" + firstPullLogOffset + + ", lastPullLogOffset=" + lastPullLogOffset + + ", channel=" + ctx.channel() + + ", opaque=" + requestHeader.getOpaque() + + '}'; + } + } +} diff --git a/qmq-server/src/main/java/qunar/tc/qmq/processor/AckMessageWorker.java b/qmq-server/src/main/java/qunar/tc/qmq/processor/AckMessageWorker.java new file mode 100644 index 00000000..655cd48a --- /dev/null +++ b/qmq-server/src/main/java/qunar/tc/qmq/processor/AckMessageWorker.java @@ -0,0 +1,58 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.processor; + +import qunar.tc.qmq.concurrent.ActorSystem; +import qunar.tc.qmq.consumer.ConsumerSequenceManager; +import qunar.tc.qmq.monitor.QMon; +import qunar.tc.qmq.protocol.CommandCode; +import qunar.tc.qmq.protocol.Datagram; +import qunar.tc.qmq.util.RemotingBuilder; + +/** + * @author yunfeng.yang + * @since 2017/7/27 + */ +class AckMessageWorker implements ActorSystem.Processor { + private final ActorSystem actorSystem; + private final ConsumerSequenceManager consumerSequenceManager; + + AckMessageWorker(final ActorSystem actorSystem, final ConsumerSequenceManager consumerSequenceManager) { + this.actorSystem = actorSystem; + this.consumerSequenceManager = consumerSequenceManager; + } + + void ack(AckMessageProcessor.AckEntry entry) { + actorSystem.dispatch("ack-" + entry.getGroup(), entry, this); + } + + @Override + public boolean process(final AckMessageProcessor.AckEntry ackEntry, ActorSystem.Actor self) { + Datagram response = RemotingBuilder.buildEmptyResponseDatagram(CommandCode.SUCCESS, ackEntry.getRequestHeader()); + try { + if (!consumerSequenceManager.putAckActions(ackEntry)) { + response = RemotingBuilder.buildEmptyResponseDatagram(CommandCode.BROKER_ERROR, ackEntry.getRequestHeader()); + } + } catch (Exception e) { + response = RemotingBuilder.buildEmptyResponseDatagram(CommandCode.BROKER_ERROR, ackEntry.getRequestHeader()); + } finally { + QMon.ackProcessTime(ackEntry.getSubject(), ackEntry.getGroup(), System.currentTimeMillis() - ackEntry.getAckBegin()); + ackEntry.getCtx().writeAndFlush(response); + } + return true; + } +} diff --git a/qmq-server/src/main/java/qunar/tc/qmq/processor/BrokerConnectionEventHandler.java b/qmq-server/src/main/java/qunar/tc/qmq/processor/BrokerConnectionEventHandler.java new file mode 100644 index 00000000..57d9413c --- /dev/null +++ b/qmq-server/src/main/java/qunar/tc/qmq/processor/BrokerConnectionEventHandler.java @@ -0,0 +1,50 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.processor; + +import io.netty.channel.ChannelHandlerContext; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import qunar.tc.qmq.monitor.QMon; +import qunar.tc.qmq.netty.ConnectionEventHandler; +import qunar.tc.qmq.stats.BrokerStats; + +/** + * @author keli.wang + * @since 2018/7/18 + */ +public class BrokerConnectionEventHandler implements ConnectionEventHandler { + private static final Logger LOG = LoggerFactory.getLogger(BrokerConnectionEventHandler.class); + + private final BrokerStats brokerStats = BrokerStats.getInstance(); + + public BrokerConnectionEventHandler() { + QMon.activeClientCount(() -> brokerStats.getActiveClientConnectionCount().doubleValue()); + } + + @Override + public void channelActive(ChannelHandlerContext ctx) { + LOG.info("client {} connected", ctx.channel().remoteAddress()); + brokerStats.getActiveClientConnectionCount().incrementAndGet(); + } + + @Override + public void channelInactive(ChannelHandlerContext ctx) { + LOG.info("client {} disconnected", ctx.channel().remoteAddress()); + brokerStats.getActiveClientConnectionCount().decrementAndGet(); + } +} diff --git a/qmq-server/src/main/java/qunar/tc/qmq/processor/GetQueueCountProcessor.java b/qmq-server/src/main/java/qunar/tc/qmq/processor/GetQueueCountProcessor.java new file mode 100644 index 00000000..2f1b5f20 --- /dev/null +++ b/qmq-server/src/main/java/qunar/tc/qmq/processor/GetQueueCountProcessor.java @@ -0,0 +1,99 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.processor; + +import io.netty.buffer.ByteBuf; +import io.netty.channel.ChannelHandlerContext; +import qunar.tc.qmq.protocol.CommandCode; +import qunar.tc.qmq.protocol.Datagram; +import qunar.tc.qmq.protocol.PayloadHolder; +import qunar.tc.qmq.protocol.RemotingCommand; +import qunar.tc.qmq.store.MessageStoreWrapper; +import qunar.tc.qmq.util.RemotingBuilder; +import qunar.tc.qmq.utils.PayloadHolderUtils; + +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.CompletableFuture; + +/** + * Created by zhaohui.yu + * 6/19/18 + */ +public class GetQueueCountProcessor extends AbstractRequestProcessor { + private final MessageStoreWrapper store; + + public GetQueueCountProcessor(MessageStoreWrapper store) { + this.store = store; + } + + @Override + public CompletableFuture processRequest(ChannelHandlerContext ctx, RemotingCommand request) { + try { + List consumers = deserialize(request); + List result = new ArrayList<>(consumers.size()); + for (Consumer consumer : consumers) { + long queueCount = store.getQueueCount(consumer.subject, consumer.group); + result.add(queueCount); + } + final Datagram response = RemotingBuilder.buildResponseDatagram(CommandCode.SUCCESS, request.getHeader(), new GetQueueCountPayloadHolder(result)); + return CompletableFuture.completedFuture(response); + } catch (Exception e) { + final Datagram response = RemotingBuilder.buildEmptyResponseDatagram(CommandCode.BROKER_ERROR, request.getHeader()); + return CompletableFuture.completedFuture(response); + } + } + + private List deserialize(RemotingCommand request) { + ByteBuf body = request.getBody(); + List consumers = new ArrayList<>(); + while (body.readableBytes() > 0) { + String subject = PayloadHolderUtils.readString(body); + String group = PayloadHolderUtils.readString(body); + consumers.add(new Consumer(subject, group)); + } + return consumers; + } + + private static class GetQueueCountPayloadHolder implements PayloadHolder { + + private final List result; + + public GetQueueCountPayloadHolder(List result) { + this.result = result; + } + + @Override + public void writeBody(ByteBuf out) { + out.writeShort((short) result.size()); + for (Long count : result) { + out.writeLong(count); + } + } + } + + private static class Consumer { + public final String subject; + + public final String group; + + private Consumer(String subject, String group) { + this.subject = subject; + this.group = group; + } + } +} diff --git a/qmq-server/src/main/java/qunar/tc/qmq/processor/PullMessageProcessor.java b/qmq-server/src/main/java/qunar/tc/qmq/processor/PullMessageProcessor.java new file mode 100644 index 00000000..d0829d09 --- /dev/null +++ b/qmq-server/src/main/java/qunar/tc/qmq/processor/PullMessageProcessor.java @@ -0,0 +1,371 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.processor; + +import com.google.common.base.CharMatcher; +import com.google.common.base.Strings; +import io.netty.buffer.ByteBuf; +import io.netty.channel.ChannelHandlerContext; +import io.netty.util.HashedWheelTimer; +import io.netty.util.Timeout; +import io.netty.util.TimerTask; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import qunar.tc.qmq.base.PullMessageResult; +import qunar.tc.qmq.concurrent.ActorSystem; +import qunar.tc.qmq.configuration.DynamicConfig; +import qunar.tc.qmq.consumer.SubscriberStatusChecker; +import qunar.tc.qmq.monitor.QMon; +import qunar.tc.qmq.protocol.*; +import qunar.tc.qmq.protocol.consumer.PullRequest; +import qunar.tc.qmq.stats.BrokerStats; +import qunar.tc.qmq.store.ConsumerLogWroteEvent; +import qunar.tc.qmq.store.MessageStoreWrapper; +import qunar.tc.qmq.store.SegmentBuffer; +import qunar.tc.qmq.store.event.FixedExecOrderEventBus; +import qunar.tc.qmq.util.RemotingBuilder; +import qunar.tc.qmq.utils.ConsumerGroupUtils; +import qunar.tc.qmq.utils.Flags; +import qunar.tc.qmq.utils.PayloadHolderUtils; + +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.TimeUnit; + +import static qunar.tc.qmq.protocol.RemotingHeader.VERSION_8; + +/** + * @author yunfeng.yang + * @since 2017/7/4 + */ +public class PullMessageProcessor extends AbstractRequestProcessor implements FixedExecOrderEventBus.Listener { + private static final Logger LOG = LoggerFactory.getLogger(PullMessageProcessor.class); + + private static final int DEFAULT_NETWORK_TIMEOUT = 5000; + private static final int DEFAULT_MAX_LOAD_TIME = 2500; + + private static final CharMatcher ILLEGAL_MATCHER = CharMatcher.anyOf("/\r\n"); + + private final HashedWheelTimer timer = new HashedWheelTimer(50, TimeUnit.MILLISECONDS); + private final DynamicConfig config; + private final ActorSystem actorSystem; + private final SubscriberStatusChecker subscriberStatusChecker; + private final PullMessageWorker pullMessageWorker; + + public PullMessageProcessor(final DynamicConfig config, + final ActorSystem actorSystem, + final MessageStoreWrapper messageStoreWrapper, + final SubscriberStatusChecker subscriberStatusChecker) { + this.config = config; + this.actorSystem = actorSystem; + this.subscriberStatusChecker = subscriberStatusChecker; + this.pullMessageWorker = new PullMessageWorker(messageStoreWrapper, actorSystem); + this.timer.start(); + } + + @Override + public CompletableFuture processRequest(final ChannelHandlerContext ctx, final RemotingCommand command) { + final PullRequest pullRequest = deserializePullRequest(command.getHeader().getVersion(), command.getBody()); + + BrokerStats.getInstance().getLastMinutePullRequestCount().add(1); + QMon.pullRequestCountInc(pullRequest.getSubject(), pullRequest.getGroup()); + + if (!checkAndRepairPullRequest(pullRequest)) { + return CompletableFuture.completedFuture(crateErrorParamResult(command)); + } + + subscribe(pullRequest); + + final PullEntry entry = new PullEntry(pullRequest, command.getHeader(), ctx); + pullMessageWorker.pull(entry); + return null; + } + + // TODO(keli.wang): how to handle broadcast subscriber correctly? + private void subscribe(PullRequest pullRequest) { + if (pullRequest.isBroadcast()) return; + + final String subject = pullRequest.getSubject(); + final String group = pullRequest.getGroup(); + final String consumerId = pullRequest.getConsumerId(); + subscriberStatusChecker.addSubscriber(subject, group, consumerId); + subscriberStatusChecker.heartbeat(consumerId, subject, group); + } + + private boolean checkAndRepairPullRequest(PullRequest pullRequest) { + final String subject = pullRequest.getSubject(); + final String group = pullRequest.getGroup(); + final String consumerId = pullRequest.getConsumerId(); + + if (Strings.isNullOrEmpty(subject) + || Strings.isNullOrEmpty(group) + || Strings.isNullOrEmpty(consumerId) + || hasIllegalPart(subject, group, consumerId)) { + QMon.pullParamErrorCountInc(subject, group); + LOG.warn("receive pull request param error, request: {}", pullRequest); + return false; + } + + if (pullRequest.getRequestNum() <= 0) { + QMon.nonPositiveRequestNumCountInc(subject, group); + + if (config.getBoolean("PullMessageProcessor.AllowNonPositiveRequestNum", false)) { + pullRequest.setRequestNum(20); + } else { + return false; + } + } + + if (pullRequest.getRequestNum() > 10000) { + pullRequest.setRequestNum(10000); + } + + return true; + } + + private boolean hasIllegalPart(final String... parts) { + for (final String part : parts) { + if (ILLEGAL_MATCHER.matchesAnyOf(part)) { + return true; + } + } + + return false; + } + + private Datagram crateErrorParamResult(RemotingCommand command) { + return RemotingBuilder.buildEmptyResponseDatagram(CommandCode.NO_MESSAGE, command.getHeader()); + } + + private PullRequest deserializePullRequest(final int version, ByteBuf input) { + String prefix = PayloadHolderUtils.readString(input); + String group = PayloadHolderUtils.readString(input); + String consumerId = PayloadHolderUtils.readString(input); + int requestNum = input.readInt(); + long offset = input.readLong(); + long pullOffsetBegin = input.readLong(); + long pullOffsetLast = input.readLong(); + long timeout = input.readLong(); + byte broadcast = input.readByte(); + + PullRequest request = new PullRequest(); + deserializeTags(request, version, input); + request.setSubject(prefix); + request.setGroup(group); + request.setConsumerId(consumerId); + request.setRequestNum(requestNum); + request.setOffset(offset); + request.setPullOffsetBegin(pullOffsetBegin); + request.setPullOffsetLast(pullOffsetLast); + request.setTimeoutMillis(timeout); + request.setBroadcast(broadcast != 0); + return request; + } + + private void deserializeTags(PullRequest request, int version, ByteBuf input) { + if (version < VERSION_8) return; + + int tagTypeCode = input.readShort(); + final byte tagSize = input.readByte(); + List tags = new ArrayList<>(tagSize); + for (int i = 0; i < tagSize; i++) { + int len = input.readShort(); + byte[] bs = new byte[len]; + input.readBytes(bs); + tags.add(bs); + } + request.setTagTypeCode(tagTypeCode); + request.setTags(tags); + } + + @Override + public void onEvent(final ConsumerLogWroteEvent e) { + if (!e.isSuccess() || Strings.isNullOrEmpty(e.getSubject())) { + return; + } + pullMessageWorker.remindNewMessages(e.getSubject()); + } + + class PullEntry implements TimerTask { + final String subject; + final String group; + final long pullBegin; + final RemotingHeader requestHeader; + final PullRequest pullRequest; + + private final ChannelHandlerContext ctx; + private final long deadline; + private volatile boolean isTimeout; + + PullEntry(PullRequest pullRequest, RemotingHeader requestHeader, ChannelHandlerContext ctx) { + this.pullRequest = pullRequest; + this.subject = pullRequest.getSubject(); + this.group = pullRequest.getGroup(); + this.requestHeader = requestHeader; + this.ctx = ctx; + this.pullBegin = System.currentTimeMillis(); + this.deadline = pullBegin + DEFAULT_NETWORK_TIMEOUT + Math.max(pullRequest.getTimeoutMillis(), 0); + } + + @Override + public void run(Timeout timeout) { + isTimeout = true; + QMon.pullTimeOutCountInc(subject, group); + actorSystem.resume(ConsumerGroupUtils.buildConsumerGroupKey(subject, group)); + } + + boolean isTimeout() { + return isTimeout; + } + + boolean expired() { + return deadline - System.currentTimeMillis() < DEFAULT_MAX_LOAD_TIME; + } + + boolean isPullOnce() { + return pullRequest.getTimeoutMillis() < 0; + } + + boolean setTimerOnDemand() { + if (pullRequest.getTimeoutMillis() <= 0) return false; + + long elapsed = System.currentTimeMillis() - pullBegin; + long wait = pullRequest.getTimeoutMillis() - elapsed; + if (wait > 0) { + timer.newTimeout(this, wait, TimeUnit.MILLISECONDS); + return true; + } + return false; + } + + void processNoMessageResult() { + QMon.pulledNoMessagesCountInc(subject, group); + + final Datagram response = RemotingBuilder.buildEmptyResponseDatagram(CommandCode.NO_MESSAGE, requestHeader); + ctx.writeAndFlush(response).addListener(future -> monitorPullProcessTime()); + } + + void processMessageResult(PullMessageResult pullMessageResult) { + if (pullMessageResult.getMessageNum() <= 0) { + processNoMessageResult(); + return; + } + + QMon.pulledMessagesCountInc(subject, group, pullMessageResult.getMessageNum()); + QMon.pulledMessageBytesCountInc(subject, group, pullMessageResult.getBufferTotalSize()); + final Datagram response = RemotingBuilder.buildResponseDatagram(CommandCode.SUCCESS, requestHeader, toPayloadHolder(pullMessageResult, requestHeader)); + ctx.writeAndFlush(response).addListener(future -> monitorPullProcessTime()); + } + + private void monitorPullProcessTime() { + QMon.pullProcessTime(subject, group, System.currentTimeMillis() - pullBegin); + } + + private PullMessageResultPayloadHolder toPayloadHolder(final PullMessageResult result, final RemotingHeader requestHeader) { + final long start = System.currentTimeMillis(); + + try { + int payloadSize = 8 + 8 + result.getBufferTotalSize(); + final ByteBuffer output = ByteBuffer.allocate(payloadSize); + output.putLong(result.getPullLogOffset()); + output.putLong(-1); + + final List buffers = result.getBuffers(); + for (final SegmentBuffer buffer : buffers) { + try { + ByteBuffer message = buffer.getBuffer(); + //新客户端拉取消息 + if (requestHeader.getVersion() >= VERSION_8) { + output.put(message); + } else { + //老客户端拉取消息 + message.mark(); + byte flag = message.get(); + //老客户端拉取消息,但是没有tag + if (!Flags.hasTags(flag)) { + message.reset(); + output.put(message); + } else { + //老客户端拉取有tag的消息 + removeTags(output, message); + } + } + } finally { + buffer.release(); + } + } + if (output.hasRemaining()) { + //将流中原来tags所占的区间释放 + return new PullMessageResultPayloadHolder(Arrays.copyOf(output.array(), output.position())); + } else { + return new PullMessageResultPayloadHolder(output.array()); + } + } finally { + QMon.readPullResultAsBytesElapsed(subject, group, System.currentTimeMillis() - start); + } + } + + private void removeTags(ByteBuffer payloadBuffer, ByteBuffer message) { + skip(message, 8 + 8); + short subjectLen = message.getShort(); + skip(message, subjectLen); + + short messageIdLen = message.getShort(); + skip(message, messageIdLen); + + int current = message.position(); + message.reset(); + int originalLimit = message.limit(); + message.limit(current); + payloadBuffer.put(message); + + message.limit(originalLimit); + message.position(current); + + skipTags(message); + payloadBuffer.put(message); + } + + private void skipTags(ByteBuffer message) { + byte tagSize = message.get(); + for (int i = 0; i < tagSize; i++) { + short tagLen = message.getShort(); + skip(message, tagLen); + } + } + + private void skip(ByteBuffer buffer, int bytes) { + buffer.position(buffer.position() + bytes); + } + } + + class PullMessageResultPayloadHolder implements PayloadHolder { + private final byte[] data; + + PullMessageResultPayloadHolder(final byte[] data) { + this.data = data; + } + + @Override + public void writeBody(ByteBuf out) { + out.writeBytes(data); + } + } +} \ No newline at end of file diff --git a/qmq-server/src/main/java/qunar/tc/qmq/processor/PullMessageWorker.java b/qmq-server/src/main/java/qunar/tc/qmq/processor/PullMessageWorker.java new file mode 100644 index 00000000..001d9bd1 --- /dev/null +++ b/qmq-server/src/main/java/qunar/tc/qmq/processor/PullMessageWorker.java @@ -0,0 +1,104 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.processor; + +import qunar.tc.qmq.base.PullMessageResult; +import qunar.tc.qmq.concurrent.ActorSystem; +import qunar.tc.qmq.monitor.QMon; +import qunar.tc.qmq.store.MessageStoreWrapper; +import qunar.tc.qmq.utils.ConsumerGroupUtils; +import qunar.tc.qmq.utils.ObjectUtils; + +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; + +/** + * @author yunfeng.yang + * @since 2017/10/30 + */ +class PullMessageWorker implements ActorSystem.Processor { + + private static final Object HOLDER = new Object(); + + private final MessageStoreWrapper store; + private final ActorSystem actorSystem; + private final ConcurrentMap> subscribers; + + PullMessageWorker(MessageStoreWrapper store, ActorSystem actorSystem) { + this.store = store; + this.actorSystem = actorSystem; + this.subscribers = new ConcurrentHashMap<>(); + } + + void pull(PullMessageProcessor.PullEntry pullEntry) { + final String actorPath = ConsumerGroupUtils.buildConsumerGroupKey(pullEntry.subject, pullEntry.group); + actorSystem.dispatch(actorPath, pullEntry, this); + } + + @Override + public boolean process(PullMessageProcessor.PullEntry entry, ActorSystem.Actor self) { + QMon.pullQueueTime(entry.subject, entry.group, entry.pullBegin); + + //开始处理请求的时候就过期了,那么就直接不处理了,也不返回任何东西给客户端,客户端等待超时 + //因为出现这种情况一般是server端排队严重,暂时挂起客户端可以避免情况恶化 + if (entry.expired()) { + QMon.pullExpiredCountInc(entry.subject, entry.group); + return true; + } + + final PullMessageResult pullMessageResult = store.findMessages(entry.pullRequest); + + if (pullMessageResult == PullMessageResult.FILTER_EMPTY || + pullMessageResult.getMessageNum() > 0 + || entry.isPullOnce() + || entry.isTimeout()) { + entry.processMessageResult(pullMessageResult); + return true; + } + + self.suspend(); + if (entry.setTimerOnDemand()) { + QMon.suspendRequestCountInc(entry.subject, entry.group); + subscribe(entry.subject, entry.group); + return false; + } + + self.resume(); + entry.processNoMessageResult(); + return true; + } + + private void subscribe(String subject, String group) { + ConcurrentMap map = subscribers.get(subject); + if (map == null) { + map = new ConcurrentHashMap<>(); + map = ObjectUtils.defaultIfNull(subscribers.putIfAbsent(subject, map), map); + } + map.putIfAbsent(group, HOLDER); + } + + void remindNewMessages(final String subject) { + final ConcurrentMap map = this.subscribers.get(subject); + if (map == null) return; + + for (String group : map.keySet()) { + map.remove(group); + this.actorSystem.resume(ConsumerGroupUtils.buildConsumerGroupKey(subject, group)); + QMon.resumeActorCountInc(subject, group); + } + } +} \ No newline at end of file diff --git a/qmq-server/src/main/java/qunar/tc/qmq/processor/SendMessageProcessor.java b/qmq-server/src/main/java/qunar/tc/qmq/processor/SendMessageProcessor.java new file mode 100644 index 00000000..c780e069 --- /dev/null +++ b/qmq-server/src/main/java/qunar/tc/qmq/processor/SendMessageProcessor.java @@ -0,0 +1,114 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.processor; + +import com.google.common.collect.Lists; +import com.google.common.util.concurrent.FutureCallback; +import com.google.common.util.concurrent.Futures; +import com.google.common.util.concurrent.ListenableFuture; +import io.netty.buffer.ByteBuf; +import io.netty.channel.ChannelHandlerContext; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import qunar.tc.qmq.base.MessageHeader; +import qunar.tc.qmq.base.RawMessage; +import qunar.tc.qmq.monitor.QMon; +import qunar.tc.qmq.protocol.CommandCode; +import qunar.tc.qmq.protocol.Datagram; +import qunar.tc.qmq.protocol.RemotingCommand; +import qunar.tc.qmq.stats.BrokerStats; +import qunar.tc.qmq.util.RemotingBuilder; + +import java.util.Collections; +import java.util.List; +import java.util.concurrent.CompletableFuture; + +import static qunar.tc.qmq.protocol.QMQSerializer.deserializeMessageHeader; + +/** + * @author yunfeng.yang + * @since 2017/7/4 + */ +public class SendMessageProcessor extends AbstractRequestProcessor { + private static final Logger LOG = LoggerFactory.getLogger(SendMessageProcessor.class); + + private final SendMessageWorker sendMessageWorker; + + public SendMessageProcessor(SendMessageWorker sendMessageWorker) { + this.sendMessageWorker = sendMessageWorker; + } + + @Override + public CompletableFuture processRequest(ChannelHandlerContext ctx, RemotingCommand command) { + List messages; + try { + messages = deserializeRawMessages(command); + } catch (Exception e) { + LOG.error("received invalid message. channel: {}", ctx.channel(), e); + QMon.brokerReceivedInvalidMessageCountInc(); + + final Datagram response = RemotingBuilder.buildEmptyResponseDatagram(CommandCode.BROKER_ERROR, command.getHeader()); + return CompletableFuture.completedFuture(response); + } + + BrokerStats.getInstance().getLastMinuteSendRequestCount().add(messages.size()); + + final ListenableFuture result = sendMessageWorker.receive(messages, command); + final CompletableFuture future = new CompletableFuture<>(); + Futures.addCallback(result, new FutureCallback() { + @Override + public void onSuccess(Datagram datagram) { + future.complete(datagram); + } + + @Override + public void onFailure(Throwable ex) { + future.completeExceptionally(ex); + } + } + ); + return future; + } + + private List deserializeRawMessages(final RemotingCommand command) { + final ByteBuf body = command.getBody(); + if (body.readableBytes() == 0) return Collections.emptyList(); + + List messages = Lists.newArrayList(); + while (body.isReadable()) { + messages.add(deserializeRawMessageWithCrc(body)); + } + return messages; + } + + private RawMessage deserializeRawMessageWithCrc(ByteBuf body) { + long bodyCrc = body.readLong(); + + int headerStart = body.readerIndex(); + body.markReaderIndex(); + MessageHeader header = deserializeMessageHeader(body); + header.setBodyCrc(bodyCrc); + int bodyLen = body.readInt(); + int headerLen = body.readerIndex() - headerStart; + + int totalLen = headerLen + bodyLen; + body.resetReaderIndex(); + ByteBuf messageBuf = body.readSlice(totalLen); + return new RawMessage(header, messageBuf, totalLen); + } + +} diff --git a/qmq-server/src/main/java/qunar/tc/qmq/processor/SendMessageWorker.java b/qmq-server/src/main/java/qunar/tc/qmq/processor/SendMessageWorker.java new file mode 100644 index 00000000..bfa7028e --- /dev/null +++ b/qmq-server/src/main/java/qunar/tc/qmq/processor/SendMessageWorker.java @@ -0,0 +1,261 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.processor; + +import com.google.common.base.CharMatcher; +import com.google.common.eventbus.Subscribe; +import com.google.common.util.concurrent.Futures; +import com.google.common.util.concurrent.ListenableFuture; +import com.google.common.util.concurrent.SettableFuture; +import io.netty.buffer.ByteBuf; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import qunar.tc.qmq.base.*; +import qunar.tc.qmq.configuration.BrokerConfig; +import qunar.tc.qmq.configuration.DynamicConfig; +import qunar.tc.qmq.monitor.QMon; +import qunar.tc.qmq.processor.filters.Invoker; +import qunar.tc.qmq.processor.filters.ReceiveFilterChain; +import qunar.tc.qmq.protocol.*; +import qunar.tc.qmq.protocol.producer.MessageProducerCode; +import qunar.tc.qmq.store.MessageStoreWrapper; +import qunar.tc.qmq.util.RemotingBuilder; +import qunar.tc.qmq.utils.CharsetUtils; +import qunar.tc.qmq.utils.RetrySubjectUtils; + +import java.util.ArrayList; +import java.util.List; +import java.util.Queue; +import java.util.concurrent.LinkedBlockingQueue; + +/** + * @author yunfeng.yang + * @since 2017/8/7 + */ +public class SendMessageWorker { + private static final Logger LOG = LoggerFactory.getLogger(SendMessageWorker.class); + + private static final CharMatcher ILLEGAL_MATCHER = CharMatcher.anyOf("/\r\n"); + + private final DynamicConfig config; + private final Invoker invoker; + private final MessageStoreWrapper messageStore; + private final Queue waitSlaveSyncQueue; + + public SendMessageWorker(final DynamicConfig config, final MessageStoreWrapper messageStore) { + this.config = config; + this.messageStore = messageStore; + this.invoker = new ReceiveFilterChain().buildFilterChain(this::doInvoke); + this.waitSlaveSyncQueue = new LinkedBlockingQueue<>(); + } + + ListenableFuture receive(final List messages, final RemotingCommand cmd) { + final List> futures = new ArrayList<>(messages.size()); + for (final RawMessage message : messages) { + final MessageHeader header = message.getHeader(); + monitorMessageReceived(header.getCreateTime(), header.getSubject()); + + final ReceivingMessage receivingMessage = new ReceivingMessage(message, cmd.getReceiveTime()); + futures.add(receivingMessage.promise()); + invoker.invoke(receivingMessage); + } + + final short version = cmd.getHeader().getVersion(); + return Futures.transform(Futures.allAsList(futures), + input -> RemotingBuilder.buildResponseDatagram(CommandCode.SUCCESS, cmd.getHeader(), new SendResultPayloadHolder(input, version))); + } + + private void monitorMessageReceived(long receiveTime, String subject) { + if (RetrySubjectUtils.isRetrySubject(subject)) { + String[] subjectAndGroup = RetrySubjectUtils.parseSubjectAndGroup(subject); + if (subjectAndGroup == null || subjectAndGroup.length != 2) return; + QMon.consumerErrorCount(subjectAndGroup[0], subjectAndGroup[1]); + return; + } + + if (RetrySubjectUtils.isDeadRetrySubject(subject)) { + String[] subjectAndGroup = RetrySubjectUtils.parseSubjectAndGroup(subject); + if (subjectAndGroup == null || subjectAndGroup.length != 2) return; + QMon.consumerErrorCount(subjectAndGroup[0], subjectAndGroup[1]); + QMon.deadLetterQueueCount(subjectAndGroup[0], subjectAndGroup[1]); + return; + } + + QMon.receivedMessagesCountInc(subject); + QMon.produceTime(subject, System.currentTimeMillis() - receiveTime); + } + + private void doInvoke(ReceivingMessage message) { + if (BrokerConfig.isReadonly()) { + brokerReadOnly(message); + return; + } + + if (bigSlaveLag()) { + brokerReadOnly(message); + return; + } + + final String subject = message.getSubject(); + if (isIllegalSubject(subject)) { + if (isRejectIllegalSubject()) { + notAllowed(message); + return; + } + } + + try { + ReceiveResult result = messageStore.putMessage(message); + offer(message, result); + } catch (Throwable t) { + error(message, t); + } + } + + private void notAllowed(ReceivingMessage message) { + QMon.rejectReceivedMessageCountInc(message.getSubject()); + end(message, new ReceiveResult(message.getMessageId(), MessageProducerCode.SUBJECT_NOT_ASSIGNED, "message rejected", -1)); + } + + private boolean isIllegalSubject(final String subject) { + if (ILLEGAL_MATCHER.matchesAnyOf(subject)) { + QMon.receivedIllegalSubjectMessagesCountInc(subject); + return true; + } + + return false; + } + + private boolean isRejectIllegalSubject() { + return config.getBoolean("Receiver.RejectIllegalSubject", true); + } + + private boolean bigSlaveLag() { + return shouldWaitSlave() && waitSlaveSyncQueue.size() >= config.getInt("receive.queue.size", 50000); + } + + private boolean shouldWaitSlave() { + return config.getBoolean("wait.slave.wrote", false); + } + + private void brokerReadOnly(ReceivingMessage message) { + QMon.brokerReadOnlyMessageCountInc(message.getSubject()); + end(message, new ReceiveResult(message.getMessageId(), MessageProducerCode.BROKER_READ_ONLY, "BROKER READ ONLY", -1)); + } + + private void error(ReceivingMessage message, Throwable e) { + LOG.error("save message error", e); + QMon.receivedFailedCountInc(message.getSubject()); + end(message, new ReceiveResult(message.getMessageId(), MessageProducerCode.STORE_ERROR, "store error", -1)); + } + + private void offer(ReceivingMessage message, ReceiveResult result) { + if (!message.isHigh()) { + end(message, result); + return; + } + if (result.getCode() != MessageProducerCode.SUCCESS) { + end(message, result); + return; + } + if (!shouldWaitSlave()) { + end(message, result); + return; + } + waitSlaveSyncQueue.offer(new ReceiveEntry(message, result)); + } + + private void end(ReceivingMessage message, ReceiveResult result) { + try { + message.done(result); + } catch (Throwable e) { + LOG.error("send response failed {}", message.getMessageId()); + } finally { + QMon.processTime(message.getSubject(), System.currentTimeMillis() - message.getReceivedTime()); + } + } + + @Subscribe + @SuppressWarnings("unused") + public void syncRequest(SyncRequest syncRequest) { + final long syncedOffset = syncRequest.getMessageLogOffset(); + ReceiveEntry first; + while ((first = this.waitSlaveSyncQueue.peek()) != null) { + if (first.result.getEndOffsetOfMessage() > syncedOffset) break; + + this.waitSlaveSyncQueue.poll(); + end(first.message, first.result); + } + } + + public static class SendResultPayloadHolder implements PayloadHolder { + private final List results; + private final short version; + + SendResultPayloadHolder(List results, short version) { + this.results = results; + this.version = version; + } + + @Override + public void writeBody(ByteBuf out) { + if (version < RemotingHeader.VERSION_4) { + writeBodyV3(out); + } else { + for (ReceiveResult result : results) { + int code = result.getCode(); + if (MessageProducerCode.SUCCESS == code) continue; + + writeItem(result, out); + } + } + } + + private void writeString(String str, ByteBuf out) { + byte[] bytes = CharsetUtils.toUTF8Bytes(str); + if (bytes != null) { + out.writeShort((short) bytes.length); + out.writeBytes(bytes); + } else { + out.writeShort(0); + } + } + + private void writeBodyV3(ByteBuf out) { + for (ReceiveResult result : results) { + writeItem(result, out); + } + } + + private void writeItem(ReceiveResult result, ByteBuf out) { + int code = result.getCode(); + writeString(result.getMessageId(), out); + out.writeInt(code); + writeString(result.getRemark(), out); + } + } + + private static class ReceiveEntry { + final ReceivingMessage message; + final ReceiveResult result; + + ReceiveEntry(ReceivingMessage message, ReceiveResult result) { + this.message = message; + this.result = result; + } + } +} diff --git a/qmq-server/src/main/java/qunar/tc/qmq/processor/filters/Invoker.java b/qmq-server/src/main/java/qunar/tc/qmq/processor/filters/Invoker.java new file mode 100644 index 00000000..531bf790 --- /dev/null +++ b/qmq-server/src/main/java/qunar/tc/qmq/processor/filters/Invoker.java @@ -0,0 +1,27 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.processor.filters; + +import qunar.tc.qmq.base.ReceivingMessage; + +/** + * @author yunfeng.yang + * @since 2017/8/7 + */ +public interface Invoker { + void invoke(ReceivingMessage message); +} diff --git a/qmq-server/src/main/java/qunar/tc/qmq/processor/filters/ReceiveFilter.java b/qmq-server/src/main/java/qunar/tc/qmq/processor/filters/ReceiveFilter.java new file mode 100644 index 00000000..d176978f --- /dev/null +++ b/qmq-server/src/main/java/qunar/tc/qmq/processor/filters/ReceiveFilter.java @@ -0,0 +1,26 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.processor.filters; + +import qunar.tc.qmq.base.ReceivingMessage; + +/** + * User: zhaohuiyu Date: 4/1/13 Time: 3:10 PM + */ +public interface ReceiveFilter { + void invoke(Invoker invoker, ReceivingMessage message); +} \ No newline at end of file diff --git a/qmq-server/src/main/java/qunar/tc/qmq/processor/filters/ReceiveFilterChain.java b/qmq-server/src/main/java/qunar/tc/qmq/processor/filters/ReceiveFilterChain.java new file mode 100644 index 00000000..a4555f8d --- /dev/null +++ b/qmq-server/src/main/java/qunar/tc/qmq/processor/filters/ReceiveFilterChain.java @@ -0,0 +1,61 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.processor.filters; + +import qunar.tc.qmq.common.Disposable; + +import java.util.ArrayList; +import java.util.List; + +/** + * User: zhaohuiyu Date: 4/2/13 Time: 12:14 PM + */ +public class ReceiveFilterChain implements Disposable { + private final List filters = new ArrayList<>(); + + public ReceiveFilterChain() { + addFilter(new ValidateFilter()); + } + + public Invoker buildFilterChain(Invoker invoker) { + Invoker last = invoker; + if (filters.size() > 0) { + for (int i = filters.size() - 1; i >= 0; i--) { + final ReceiveFilter filter = filters.get(i); + final Invoker next = last; + last = message -> filter.invoke(next, message); + } + } + return last; + } + + private void addFilter(ReceiveFilter filter) { + filters.add(filter); + } + + @Override + public void destroy() { + if (filters != null && !filters.isEmpty()) { + for (ReceiveFilter filter : filters) { + if (filter instanceof Disposable) { + ((Disposable) filter).destroy(); + } + } + } + } +} + diff --git a/qmq-server/src/main/java/qunar/tc/qmq/processor/filters/ValidateFilter.java b/qmq-server/src/main/java/qunar/tc/qmq/processor/filters/ValidateFilter.java new file mode 100644 index 00000000..a398c190 --- /dev/null +++ b/qmq-server/src/main/java/qunar/tc/qmq/processor/filters/ValidateFilter.java @@ -0,0 +1,35 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.processor.filters; + +import com.google.common.base.Preconditions; +import com.google.common.base.Strings; +import qunar.tc.qmq.base.ReceivingMessage; + +/** + * User: zhaohuiyu Date: 4/1/13 Time: 7:01 PM + */ +class ValidateFilter implements ReceiveFilter { + + @Override + public void invoke(Invoker invoker, ReceivingMessage message) { + Preconditions.checkNotNull(message, "message not null"); + Preconditions.checkArgument(!Strings.isNullOrEmpty(message.getMessageId()), "message id should not be empty"); + Preconditions.checkArgument(!Strings.isNullOrEmpty(message.getSubject()), "message subject should not be empty"); + invoker.invoke(message); + } +} \ No newline at end of file diff --git a/qmq-server/src/main/java/qunar/tc/qmq/startup/ServerWrapper.java b/qmq-server/src/main/java/qunar/tc/qmq/startup/ServerWrapper.java new file mode 100644 index 00000000..23d17743 --- /dev/null +++ b/qmq-server/src/main/java/qunar/tc/qmq/startup/ServerWrapper.java @@ -0,0 +1,275 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.startup; + +import com.google.common.base.Preconditions; +import io.netty.buffer.ByteBuf; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import qunar.tc.qmq.common.Disposable; +import qunar.tc.qmq.concurrent.ActorSystem; +import qunar.tc.qmq.concurrent.NamedThreadFactory; +import qunar.tc.qmq.configuration.BrokerConfig; +import qunar.tc.qmq.configuration.DynamicConfig; +import qunar.tc.qmq.consumer.ConsumerSequenceManager; +import qunar.tc.qmq.consumer.OfflineActionHandler; +import qunar.tc.qmq.consumer.SubscriberStatusChecker; +import qunar.tc.qmq.meta.BrokerRegisterService; +import qunar.tc.qmq.meta.BrokerRole; +import qunar.tc.qmq.meta.MetaServerLocator; +import qunar.tc.qmq.netty.NettyServer; +import qunar.tc.qmq.processor.*; +import qunar.tc.qmq.protocol.CommandCode; +import qunar.tc.qmq.protocol.Datagram; +import qunar.tc.qmq.store.*; +import qunar.tc.qmq.sync.*; +import qunar.tc.qmq.sync.master.MasterSyncNettyServer; + +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; + +import static qunar.tc.qmq.constants.BrokerConstants.*; + + +/** + * @author yunfeng.yang + * @since 2017/6/30 + */ +public class ServerWrapper implements Disposable { + private static final Logger LOG = LoggerFactory.getLogger(ServerWrapper.class); + + private final DynamicConfig config; + private final List resources; + + private Integer listenPort; + private MessageStoreWrapper messageStoreWrapper; + private Storage storage; + private SendMessageWorker sendMessageWorker; + private ConsumerSequenceManager consumerSequenceManager; + private ExecutorService sendMessageExecutorService; + private ExecutorService consumeManageExecutorService; + + private SlaveSyncClient slaveSyncClient; + private MasterSyncNettyServer masterSyncNettyServer; + private MasterSlaveSyncManager masterSlaveSyncManager; + private BrokerRegisterService brokerRegisterService; + + private SubscriberStatusChecker subscriberStatusChecker; + + private NettyServer nettyServer; + + public ServerWrapper(final DynamicConfig config) { + this.config = config; + this.resources = new ArrayList<>(); + } + + public void start() { + LOG.info("qmq server init started"); + register(); + createStorage(); + startSyncLog(); + initStorage(); + startServeSync(); + startServerHandlers(); + startConsumerChecker(); + addToResources(); + online(); + LOG.info("qmq server init done"); + } + + private void register() { + this.listenPort = config.getInt(PORT_CONFIG, DEFAULT_PORT); + + final MetaServerLocator metaServerLocator = new MetaServerLocator(config.getString(META_SERVER_ENDPOINT)); + brokerRegisterService = new BrokerRegisterService(listenPort, metaServerLocator); + brokerRegisterService.start(); + + Preconditions.checkState(BrokerConfig.getBrokerRole() != BrokerRole.STANDBY, "目前broker不允许被指定为standby模式"); + } + + private void startConsumerChecker() { + if (BrokerConfig.getBrokerRole() == BrokerRole.MASTER) { + subscriberStatusChecker.start(); + } + } + + private void createStorage() { + slaveSyncClient = new SlaveSyncClient(config); + this.storage = new DefaultStorage(BrokerConfig.getBrokerRole(), new StorageConfigImpl(config), new CheckpointLoader() { + @Override + public ByteBuf loadCheckpoint() { + Datagram datagram = null; + try { + datagram = syncCheckpointUntilSuccess(); + final ByteBuf body = datagram.getBody(); + body.retain(); + return body; + } finally { + if (datagram != null) { + datagram.release(); + } + } + } + + private Datagram syncCheckpointUntilSuccess() { + while (true) { + try { + return slaveSyncClient.syncCheckpoint(); + } catch (Exception e) { + LOG.warn("sync checkpoint failed, will retry after 2 seconds", e); + try { + TimeUnit.SECONDS.sleep(2); + } catch (InterruptedException ignore) { + LOG.debug("sync checkpoint interrupted"); + } + } + } + } + }); + } + + private void startSyncLog() { + if (BrokerConfig.getBrokerRole() == BrokerRole.SLAVE) { + this.masterSlaveSyncManager = new MasterSlaveSyncManager(slaveSyncClient); + this.masterSlaveSyncManager.registerProcessor(SyncType.message, new SyncMessageLogProcessor(storage)); + this.masterSlaveSyncManager.registerProcessor(SyncType.action, new SyncActionLogProcessor(storage)); + this.masterSlaveSyncManager.registerProcessor(SyncType.heartbeat, new HeartbeatProcessor(storage)); + this.masterSlaveSyncManager.startSync(); + waitUntilSyncDone(); + } + } + + private void waitUntilSyncDone() { + final CheckpointManager checkpointManager = storage.getCheckpointManager(); + final long messageCheckpointOffset = checkpointManager.getMessageCheckpointOffset(); + final long actionCheckpointOffset = checkpointManager.getActionCheckpointOffset(); + while (true) { + final long maxMessageOffset = storage.getMaxMessageOffset(); + final long maxActionLogOffset = storage.getMaxActionLogOffset(); + + if (maxMessageOffset >= messageCheckpointOffset && maxActionLogOffset >= actionCheckpointOffset) { + return; + } + + LOG.info("waiting log sync done ..."); + try { + TimeUnit.SECONDS.sleep(2); + } catch (InterruptedException ignore) { + LOG.debug("sleep interrupted"); + } + } + } + + private void initStorage() { + this.consumerSequenceManager = new ConsumerSequenceManager(storage); + this.subscriberStatusChecker = new SubscriberStatusChecker(config, storage, consumerSequenceManager); + this.subscriberStatusChecker.init(); + this.messageStoreWrapper = new MessageStoreWrapper(storage, consumerSequenceManager); + final OfflineActionHandler handler = new OfflineActionHandler(storage); + this.storage.registerActionEventListener(handler); + this.storage.start(); + // make sure init this after storage started + this.consumerSequenceManager.init(); + + this.sendMessageExecutorService = new ThreadPoolExecutor(1, 1, + 0L, TimeUnit.MILLISECONDS, new LinkedBlockingQueue<>(), new NamedThreadFactory("send-message-processor")); + this.consumeManageExecutorService = new ThreadPoolExecutor(1, 1, + 0L, TimeUnit.MILLISECONDS, new LinkedBlockingQueue<>(), new NamedThreadFactory("consume-manage-processor")); + + this.sendMessageWorker = new SendMessageWorker(config, messageStoreWrapper); + } + + private void startServeSync() { + this.masterSyncNettyServer = new MasterSyncNettyServer(config, storage); + this.masterSyncNettyServer.registerSyncEvent(sendMessageWorker); + this.masterSyncNettyServer.start(); + } + + private void startServerHandlers() { + final ActorSystem actorSystem = new ActorSystem("qmq"); + + final PullMessageProcessor pullMessageProcessor = new PullMessageProcessor(config, actorSystem, messageStoreWrapper, subscriberStatusChecker); + this.storage.registerEventListener(ConsumerLogWroteEvent.class, pullMessageProcessor); + final SendMessageProcessor sendMessageProcessor = new SendMessageProcessor(sendMessageWorker); + final AckMessageProcessor ackMessageProcessor = new AckMessageProcessor(actorSystem, consumerSequenceManager, subscriberStatusChecker); + + this.nettyServer = new NettyServer("broker", Runtime.getRuntime().availableProcessors(), listenPort, new BrokerConnectionEventHandler()); + this.nettyServer.registerProcessor(CommandCode.SEND_MESSAGE, sendMessageProcessor, sendMessageExecutorService); + this.nettyServer.registerProcessor(CommandCode.PULL_MESSAGE, pullMessageProcessor); + this.nettyServer.registerProcessor(CommandCode.ACK_REQUEST, ackMessageProcessor); + this.nettyServer.start(); + } + + private void addToResources() { + this.resources.add(subscriberStatusChecker); + this.resources.add(brokerRegisterService); + this.resources.add(masterSyncNettyServer); + if (BrokerConfig.getBrokerRole() == BrokerRole.SLAVE) { + this.resources.add(masterSlaveSyncManager); + } + this.resources.add(nettyServer); + this.resources.add(storage); + } + + private void online() { + BrokerConfig.markAsWritable(); + brokerRegisterService.healthSwitch(true); + subscriberStatusChecker.brokerStatusChanged(true); + } + + @Override + public void destroy() { + offline(); + if (sendMessageExecutorService != null) { + try { + sendMessageExecutorService.awaitTermination(5, TimeUnit.SECONDS); + } catch (InterruptedException e) { + LOG.error("Shutdown sendMessageExecutorService interrupted."); + } + } + if (consumeManageExecutorService != null) { + try { + consumeManageExecutorService.awaitTermination(5, TimeUnit.SECONDS); + } catch (InterruptedException e) { + LOG.error("Shutdown consumeManageExecutorService interrupted."); + } + } + if (resources.isEmpty()) return; + + for (final Disposable resource : resources) { + try { + resource.destroy(); + } catch (Throwable e) { + LOG.error("destroy resource failed", e); + } + } + } + + private void offline() { + for (int i = 0; i < 3; ++i) { + try { + brokerRegisterService.healthSwitch(false); + } catch (Exception e) { + LOG.error("offline broker failed.", e); + } + } + } +} diff --git a/qmq-server/src/main/java/qunar/tc/qmq/stats/BrokerStats.java b/qmq-server/src/main/java/qunar/tc/qmq/stats/BrokerStats.java new file mode 100644 index 00000000..dcf56415 --- /dev/null +++ b/qmq-server/src/main/java/qunar/tc/qmq/stats/BrokerStats.java @@ -0,0 +1,56 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.stats; + +import java.util.concurrent.atomic.AtomicLong; + +/** + * @author keli.wang + * @since 2018/7/18 + */ +public class BrokerStats { + private static final BrokerStats STATS = new BrokerStats(); + + private final AtomicLong activeConnectionCount = new AtomicLong(0); + + private final PerMinuteDeltaCounter lastMinuteSendRequestCount = new PerMinuteDeltaCounter(); + private final PerMinuteDeltaCounter lastMinutePullRequestCount = new PerMinuteDeltaCounter(); + private final PerMinuteDeltaCounter lastMinuteAckRequestCount = new PerMinuteDeltaCounter(); + + private BrokerStats() { + } + + public static BrokerStats getInstance() { + return STATS; + } + + public AtomicLong getActiveClientConnectionCount() { + return activeConnectionCount; + } + + public PerMinuteDeltaCounter getLastMinuteSendRequestCount() { + return lastMinuteSendRequestCount; + } + + public PerMinuteDeltaCounter getLastMinutePullRequestCount() { + return lastMinutePullRequestCount; + } + + public PerMinuteDeltaCounter getLastMinuteAckRequestCount() { + return lastMinuteAckRequestCount; + } +} diff --git a/qmq-server/src/main/java/qunar/tc/qmq/stats/PerMinuteDeltaCounter.java b/qmq-server/src/main/java/qunar/tc/qmq/stats/PerMinuteDeltaCounter.java new file mode 100644 index 00000000..0080490f --- /dev/null +++ b/qmq-server/src/main/java/qunar/tc/qmq/stats/PerMinuteDeltaCounter.java @@ -0,0 +1,74 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.stats; + +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicLong; + +/** + * @author keli.wang + * @since 2018/7/19 + */ +public class PerMinuteDeltaCounter { + private final AtomicLong value; + private final long resetIntervalMillis; + private final AtomicLong nextResetTimeMillisRef; + + public PerMinuteDeltaCounter() { + this.value = new AtomicLong(); + this.resetIntervalMillis = TimeUnit.MINUTES.toMillis(1); + this.nextResetTimeMillisRef = new AtomicLong(now() + resetIntervalMillis); + } + + private long now() { + return System.currentTimeMillis(); + } + + public void add(final long delta) { + while (true) { + final long nextResetTimeMillis = nextResetTimeMillisRef.get(); + final long currentTimeMillis = now(); + if (currentTimeMillis < nextResetTimeMillis) { + value.addAndGet(delta); + return; + } + final long currentValue = value.get(); + if (nextResetTimeMillisRef.compareAndSet(nextResetTimeMillis, Long.MAX_VALUE)) { + value.addAndGet(delta - currentValue); + nextResetTimeMillisRef.set(currentTimeMillis + resetIntervalMillis); + return; + } + } + } + + public long getSum() { + while (true) { + final long nextResetTimeMillis = nextResetTimeMillisRef.get(); + final long currentValue = value.get(); + final long currentTimeMillis = now(); + if (currentTimeMillis < nextResetTimeMillis) { + return currentValue; + } + + if (nextResetTimeMillisRef.compareAndSet(nextResetTimeMillis, Long.MAX_VALUE)) { + value.addAndGet(-currentValue); + nextResetTimeMillisRef.set(currentTimeMillis + resetIntervalMillis); + return value.get(); + } + } + } +} diff --git a/qmq-server/src/main/java/qunar/tc/qmq/store/MessageStoreWrapper.java b/qmq-server/src/main/java/qunar/tc/qmq/store/MessageStoreWrapper.java new file mode 100644 index 00000000..bb1e7a56 --- /dev/null +++ b/qmq-server/src/main/java/qunar/tc/qmq/store/MessageStoreWrapper.java @@ -0,0 +1,364 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.store; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import qunar.tc.qmq.TagType; +import qunar.tc.qmq.base.*; +import qunar.tc.qmq.consumer.ConsumerSequenceManager; +import qunar.tc.qmq.monitor.QMon; +import qunar.tc.qmq.protocol.consumer.PullRequest; +import qunar.tc.qmq.protocol.producer.MessageProducerCode; +import qunar.tc.qmq.store.action.RangeAckAction; + +import java.util.ArrayList; +import java.util.List; + +import static qunar.tc.qmq.store.Tags.match; + +/** + * @author yunfeng.yang + * @since 2017/7/10 + */ +public class MessageStoreWrapper { + private static final Logger LOG = LoggerFactory.getLogger(MessageStoreWrapper.class); + + private final Storage storage; + private final ConsumerSequenceManager consumerSequenceManager; + + public MessageStoreWrapper(final Storage storage, final ConsumerSequenceManager consumerSequenceManager) { + this.storage = storage; + this.consumerSequenceManager = consumerSequenceManager; + } + + public ReceiveResult putMessage(final ReceivingMessage message) { + final RawMessage rawMessage = message.getMessage(); + final MessageHeader header = rawMessage.getHeader(); + final String msgId = header.getMessageId(); + final long start = System.currentTimeMillis(); + try { + final PutMessageResult putMessageResult = storage.appendMessage(rawMessage); + final PutMessageStatus status = putMessageResult.getStatus(); + if (status != PutMessageStatus.SUCCESS) { + LOG.error("put message error, message:{} {}, status:{}", header.getSubject(), msgId, status.name()); + QMon.storeMessageErrorCountInc(header.getSubject()); + return new ReceiveResult(msgId, MessageProducerCode.STORE_ERROR, status.name(), -1); + } + + AppendMessageResult result = putMessageResult.getResult(); + final long endOffsetOfMessage = result.getWroteOffset() + result.getWroteBytes(); + return new ReceiveResult(msgId, MessageProducerCode.SUCCESS, "", endOffsetOfMessage); + } catch (Throwable e) { + LOG.error("put message error, message:{} {}", header.getSubject(), header.getMessageId(), e); + QMon.storeMessageErrorCountInc(header.getSubject()); + return new ReceiveResult(msgId, MessageProducerCode.STORE_ERROR, "", -1); + } finally { + QMon.putMessageTime(header.getSubject(), System.currentTimeMillis() - start); + } + } + + public PullMessageResult findMessages(final PullRequest pullRequest) { + try { + final PullMessageResult unAckMessages = findUnAckMessages(pullRequest); + if (unAckMessages.getMessageNum() > 0) { + return unAckMessages; + } + + return findNewExistMessages(pullRequest); + } catch (Throwable e) { + LOG.error("find messages error, consumer: {}", pullRequest, e); + QMon.findMessagesErrorCountInc(pullRequest.getSubject(), pullRequest.getGroup()); + } + return PullMessageResult.EMPTY; + } + + private PullMessageResult findNewExistMessages(final PullRequest pullRequest) { + final String subject = pullRequest.getSubject(); + final String group = pullRequest.getGroup(); + final String consumerId = pullRequest.getConsumerId(); + final boolean isBroadcast = pullRequest.isBroadcast(); + + final long start = System.currentTimeMillis(); + try { + ConsumeQueue consumeQueue = storage.locateConsumeQueue(subject, group); + final GetMessageResult getMessageResult = consumeQueue.pollMessages(pullRequest.getRequestNum()); + switch (getMessageResult.getStatus()) { + case SUCCESS: + if (getMessageResult.getMessageNum() == 0) { + consumeQueue.setNextSequence(getMessageResult.getNextBeginOffset()); + return PullMessageResult.EMPTY; + } + + if (noRequestTag(pullRequest)) { + final WritePutActionResult writeResult = consumerSequenceManager.putPullActions(subject, group, consumerId, isBroadcast, getMessageResult); + if (writeResult.isSuccess()) { + consumeQueue.setNextSequence(getMessageResult.getNextBeginOffset()); + return new PullMessageResult(writeResult.getPullLogOffset(), getMessageResult.getSegmentBuffers(), getMessageResult.getBufferTotalSize(), getMessageResult.getMessageNum()); + } else { + getMessageResult.release(); + return PullMessageResult.EMPTY; + } + } + + return filterByTags(pullRequest, getMessageResult, consumeQueue); + case OFFSET_OVERFLOW: + LOG.warn("get message result not success, consumer:{}, result:{}", pullRequest, getMessageResult); + QMon.getMessageOverflowCountInc(subject, group); + default: + consumeQueue.setNextSequence(getMessageResult.getNextBeginOffset()); + return PullMessageResult.EMPTY; + } + } finally { + QMon.findNewExistMessageTime(subject, group, System.currentTimeMillis() - start); + } + } + + private boolean noRequestTag(PullRequest pullRequest) { + int tagTypeCode = pullRequest.getTagTypeCode(); + if (TagType.NO_TAG.getCode() == tagTypeCode) return true; + List tags = pullRequest.getTags(); + return tags == null || tags.isEmpty(); + } + + private PullMessageResult filterByTags(PullRequest pullRequest, GetMessageResult getMessageResult, ConsumeQueue consumeQueue) { + final String subject = pullRequest.getSubject(); + final String group = pullRequest.getGroup(); + final String consumerId = pullRequest.getConsumerId(); + final boolean isBroadcast = pullRequest.isBroadcast(); + + shiftRight(getMessageResult); + List filterResult = filter(getMessageResult, pullRequest.getTags(), pullRequest.getTagTypeCode()); + List retList = new ArrayList<>(); + int index; + for (index = 0; index < filterResult.size(); ++index) { + GetMessageResult item = filterResult.get(index); + if (!putAction(item, consumeQueue, subject, group, consumerId, isBroadcast, retList)) break; + } + releaseRemain(index, filterResult); + if (retList.isEmpty()) return PullMessageResult.FILTER_EMPTY; + return merge(retList); + } + + private List filter(GetMessageResult input, List tags, int tagType) { + List result = new ArrayList<>(); + + List messages = input.getSegmentBuffers(); + OffsetRange offsetRange = input.getConsumerLogRange(); + + GetMessageResult range = null; + long begin = -1; + long end = -1; + for (int i = 0; i < messages.size(); ++i) { + SegmentBuffer message = messages.get(i); + if (match(message, tags, tagType)) { + if (range == null) { + range = new GetMessageResult(); + result.add(range); + begin = offsetRange.getBegin() + i; + } + end = offsetRange.getBegin() + i; + range.addSegmentBuffer(message); + } else { + message.release(); + setOffsetRange(range, begin, end); + range = null; + } + } + setOffsetRange(range, begin, end); + appendEmpty(end, offsetRange, result); + return result; + } + + private void setOffsetRange(GetMessageResult input, long begin, long end) { + if (input != null) { + input.setConsumerLogRange(new OffsetRange(begin, end)); + input.setNextBeginOffset(end + 1); + } + } + + /* + begin=0 end=8 + ------------------------------------ + | - | - | - | + | + | + | + | + | + | + ------------------------------------- + shift -> begin=3, end=8 + */ + private void shiftRight(GetMessageResult getMessageResult) { + OffsetRange offsetRange = getMessageResult.getConsumerLogRange(); + long expectedBegin = offsetRange.getEnd() - getMessageResult.getMessageNum() + 1; + if (expectedBegin == offsetRange.getBegin()) return; + getMessageResult.setConsumerLogRange(new OffsetRange(expectedBegin, offsetRange.getEnd())); + } + + private boolean putAction(GetMessageResult range, ConsumeQueue consumeQueue, + String subject, String group, String consumerId, boolean isBroadcast, + List retList) { + final WritePutActionResult writeResult = consumerSequenceManager.putPullActions(subject, group, consumerId, isBroadcast, range); + if (writeResult.isSuccess()) { + consumeQueue.setNextSequence(range.getNextBeginOffset()); + retList.add(new PullMessageResult(writeResult.getPullLogOffset(), range.getSegmentBuffers(), range.getBufferTotalSize(), range.getMessageNum())); + return true; + } + return false; + } + + private PullMessageResult merge(List list) { + if (list.size() == 1) return list.get(0); + + long pullLogOffset = list.get(0).getPullLogOffset(); + List buffers = new ArrayList<>(); + int bufferTotalSize = 0; + int messageNum = 0; + for (PullMessageResult result : list) { + bufferTotalSize += result.getBufferTotalSize(); + messageNum += result.getMessageNum(); + buffers.addAll(result.getBuffers()); + } + return new PullMessageResult(pullLogOffset, buffers, bufferTotalSize, messageNum); + } + + private void appendEmpty(long end, OffsetRange offsetRange, List list) { + if (end < offsetRange.getEnd()) { + GetMessageResult emptyRange = new GetMessageResult(); + long begin = end == -1 ? offsetRange.getBegin() : end; + emptyRange.setConsumerLogRange(new OffsetRange(begin, offsetRange.getEnd())); + emptyRange.setNextBeginOffset(offsetRange.getEnd() + 1); + list.add(emptyRange); + } + } + + private void releaseRemain(int startIndex, List list) { + for (int i = startIndex; i < list.size(); ++i) { + list.get(i).release(); + } + } + + private PullMessageResult findUnAckMessages(final PullRequest pullRequest) { + final long start = System.currentTimeMillis(); + try { + return doFindUnAckMessages(pullRequest); + } finally { + QMon.findLostMessagesTime(pullRequest.getSubject(), pullRequest.getGroup(), System.currentTimeMillis() - start); + } + } + + private PullMessageResult doFindUnAckMessages(final PullRequest pullRequest) { + final String subject = pullRequest.getSubject(); + final String group = pullRequest.getGroup(); + final String consumerId = pullRequest.getConsumerId(); + final ConsumerSequence consumerSequence = consumerSequenceManager.getOrCreateConsumerSequence(subject, group, consumerId); + + final long ackSequence = consumerSequence.getAckSequence(); + long pullLogSequenceInConsumer = pullRequest.getPullOffsetLast(); + if (pullLogSequenceInConsumer < ackSequence) { + pullLogSequenceInConsumer = ackSequence; + } + + final long pullLogSequenceInServer = consumerSequence.getPullSequence(); + if (pullLogSequenceInServer <= pullLogSequenceInConsumer) { + return PullMessageResult.EMPTY; + } + + LOG.warn("consumer need find lost ack messages, pullRequest: {}, consumerSequence: {}", pullRequest, consumerSequence); + + final int requestNum = pullRequest.getRequestNum(); + final List buffers = new ArrayList<>(requestNum); + long firstValidSeq = -1; + int totalSize = 0; + final long firstLostAckPullLogSeq = pullLogSequenceInConsumer + 1; + for (long seq = firstLostAckPullLogSeq; buffers.size() < requestNum && seq <= pullLogSequenceInServer; seq++) { + try { + final long consumerLogSequence = getConsumerLogSequence(pullRequest, seq); + //deleted message + if (consumerLogSequence < 0) { + LOG.warn("find no consumer log for this pull log sequence, req: {}, pullLogSeq: {}, consumerLogSeq: {}", pullRequest, seq, consumerLogSequence); + if (firstValidSeq == -1) { + continue; + } else { + break; + } + } + + final GetMessageResult getMessageResult = storage.getMessage(subject, consumerLogSequence); + if (getMessageResult.getStatus() != GetMessageStatus.SUCCESS || getMessageResult.getMessageNum() == 0) { + LOG.error("getMessageResult error, consumer:{}, consumerLogSequence:{}, pullLogSequence:{}, getMessageResult:{}", pullRequest, consumerLogSequence, seq, getMessageResult); + QMon.getMessageErrorCountInc(subject, group); + if (firstValidSeq == -1) { + continue; + } else { + break; + } + } + + //re-filter un-ack message + final SegmentBuffer segmentBuffer = getMessageResult.getSegmentBuffers().get(0); + if (!noRequestTag(pullRequest) && !match(segmentBuffer, pullRequest.getTags(), pullRequest.getTagTypeCode())) { + if (firstValidSeq != -1) { + break; + } + } + + if (firstValidSeq == -1) { + firstValidSeq = seq; + } + + + buffers.add(segmentBuffer); + totalSize += segmentBuffer.getSize(); + } catch (Exception e) { + LOG.error("error occurs when find messages by pull log offset, request: {}, consumerSequence: {}", pullRequest, consumerSequence, e); + QMon.getMessageErrorCountInc(subject, group); + + if (firstValidSeq != -1) { + break; + } + } + } + + if (buffers.size() > 0) { + //说明pull log里有一段对应的消息已经被清理掉了,需要调整一下位置 + if (firstValidSeq > firstLostAckPullLogSeq) { + consumerSequence.setAckSequence(firstValidSeq - 1); + } + } else { + LOG.error("find lost messages empty, consumer:{}, consumerSequence:{}, pullLogSequence:{}", pullRequest, consumerSequence, pullLogSequenceInServer); + QMon.findLostMessageEmptyCountInc(subject, group); + firstValidSeq = pullLogSequenceInServer; + consumerSequence.setAckSequence(pullLogSequenceInServer); + + // auto ack all deleted pulled message + LOG.info("auto ack deleted pulled message. subject: {}, group: {}, consumerId: {}, firstSeq: {}, lastSeq: {}", + subject, group, consumerId, firstLostAckPullLogSeq, firstValidSeq); + consumerSequenceManager.putAction(new RangeAckAction(subject, group, consumerId, System.currentTimeMillis(), firstLostAckPullLogSeq, firstValidSeq)); + } + + final PullMessageResult result = new PullMessageResult(firstValidSeq, buffers, totalSize, buffers.size()); + QMon.findLostMessageCountInc(subject, group, result.getMessageNum()); + LOG.info("found lost ack messages request: {}, consumerSequence: {}, result: {}", pullRequest, consumerSequence, result); + return result; + } + + private long getConsumerLogSequence(PullRequest pullRequest, long offset) { + if (pullRequest.isBroadcast()) return offset; + return storage.getMessageSequenceByPullLog(pullRequest.getSubject(), pullRequest.getGroup(), pullRequest.getConsumerId(), offset); + } + + public long getQueueCount(String subject, String group) { + return storage.locateConsumeQueue(subject, group).getQueueCount(); + } +} diff --git a/qmq-server/src/main/java/qunar/tc/qmq/store/Tags.java b/qmq-server/src/main/java/qunar/tc/qmq/store/Tags.java new file mode 100644 index 00000000..46eb4b7b --- /dev/null +++ b/qmq-server/src/main/java/qunar/tc/qmq/store/Tags.java @@ -0,0 +1,89 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.store; + +import qunar.tc.qmq.TagType; +import qunar.tc.qmq.utils.Flags; +import qunar.tc.qmq.utils.ListUtils; + +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.List; + +class Tags { + static boolean match(SegmentBuffer result, List requestTags, int tagTypeCode) { + ByteBuffer message = result.getBuffer(); + message.mark(); + byte flag = message.get(); + if (!Flags.hasTags(flag)) { + message.reset(); + return false; + } + skip(message, 8 + 8); + //subject + skipString(message); + //message id + skipString(message); + + final byte tagsSize = message.get(); + if (tagsSize == 1) { + final short len = message.getShort(); + final byte[] tag = new byte[len]; + message.get(tag); + message.reset(); + return matchOneTag(tag, requestTags, tagTypeCode); + } + List tags = new ArrayList<>(tagsSize); + for (int i = 0; i < tagsSize; i++) { + final short len = message.getShort(); + final byte[] bs = new byte[len]; + message.get(bs); + tags.add(bs); + } + message.reset(); + return matchTags(tags, requestTags, tagTypeCode); + } + + private static void skipString(ByteBuffer input) { + final short len = input.getShort(); + skip(input, len); + } + + private static void skip(ByteBuffer input, int bytes) { + input.position(input.position() + bytes); + } + + private static boolean matchOneTag(byte[] tag, List requestTags, int tagTypeCode) { + if (requestTags.size() == 1 || TagType.OR.getCode() == tagTypeCode) { + return ListUtils.contains(requestTags, tag); + } + + if (TagType.AND.getCode() == tagTypeCode && requestTags.size() > 1) return false; + + return false; + } + + private static boolean matchTags(List messageTags, List requestTags, int tagTypeCode) { + if (tagTypeCode == TagType.AND.getCode()) { + return ListUtils.containsAll(messageTags, requestTags); + } + if (tagTypeCode == TagType.OR.getCode()) { + return ListUtils.intersection(messageTags, requestTags); + } + return false; + } +} diff --git a/qmq-server/src/main/java/qunar/tc/qmq/sync/master/AbstractLogSyncWorker.java b/qmq-server/src/main/java/qunar/tc/qmq/sync/master/AbstractLogSyncWorker.java new file mode 100644 index 00000000..57dbc359 --- /dev/null +++ b/qmq-server/src/main/java/qunar/tc/qmq/sync/master/AbstractLogSyncWorker.java @@ -0,0 +1,184 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.sync.master; + +import io.netty.buffer.ByteBuf; +import io.netty.util.Timeout; +import io.netty.util.TimerTask; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import qunar.tc.qmq.meta.BrokerRole; +import qunar.tc.qmq.base.SyncRequest; +import qunar.tc.qmq.configuration.BrokerConfig; +import qunar.tc.qmq.configuration.DynamicConfig; +import qunar.tc.qmq.protocol.CommandCode; +import qunar.tc.qmq.protocol.Datagram; +import qunar.tc.qmq.protocol.PayloadHolder; +import qunar.tc.qmq.protocol.RemotingHeader; +import qunar.tc.qmq.store.DataTransfer; +import qunar.tc.qmq.store.LogSegment; +import qunar.tc.qmq.store.SegmentBuffer; +import qunar.tc.qmq.sync.SyncType; +import qunar.tc.qmq.util.RemotingBuilder; +import qunar.tc.qmq.utils.HeaderSerializer; +import qunar.tc.qmq.utils.ServerTimerUtil; + +import java.nio.ByteBuffer; +import java.util.concurrent.TimeUnit; + +/** + * @author yunfeng.yang + * @since 2017/8/21 + */ +abstract class AbstractLogSyncWorker implements SyncProcessor { + private static final Logger LOG = LoggerFactory.getLogger(AbstractLogSyncWorker.class); + + private static final int SYNC_HEADER_LEN = 4 /*size*/ + 8 /*startOffset*/; + + private final DynamicConfig config; + private volatile LogSegment currentSegment; + + AbstractLogSyncWorker(final DynamicConfig config) { + this.config = config; + this.currentSegment = null; + } + + @Override + public void process(SyncRequestEntry entry) { + final SyncRequest syncRequest = entry.getSyncRequest(); + final SegmentBuffer result = getSyncLog(syncRequest); + if (result == null || result.getSize() <= 0) { + final long timeout = config.getLong("message.sync.timeout.ms", 10L); + ServerTimerUtil.newTimeout(new SyncRequestTimeoutTask(entry, this), timeout, TimeUnit.MILLISECONDS); + return; + } + + processSyncLog(entry, result); + } + + @Override + public void processTimeout(SyncRequestEntry entry) { + final SyncRequest syncRequest = entry.getSyncRequest(); + final SegmentBuffer result = getSyncLog(syncRequest); + + if (result == null || result.getSize() <= 0) { + long offset = syncRequest.getSyncType() == SyncType.message.getCode() ? syncRequest.getMessageLogOffset() : syncRequest.getActionLogOffset(); + writeEmpty(entry, offset); + return; + } + + processSyncLog(entry, result); + } + + protected abstract SegmentBuffer getSyncLog(SyncRequest syncRequest); + + private void writeEmpty(SyncRequestEntry entry, long offset) { + final SyncLogPayloadHolder payloadHolder = new SyncLogPayloadHolder(offset); + final Datagram datagram = RemotingBuilder.buildResponseDatagram(CommandCode.SUCCESS, entry.getRequestHeader(), payloadHolder); + entry.getCtx().writeAndFlush(datagram); + } + + private void processSyncLog(SyncRequestEntry entry, SegmentBuffer result) { + if (!acquireSegmentLock(result)) { + writeEmpty(entry, result.getStartOffset()); + return; + } + + if (!result.retain()) { + writeEmpty(entry, result.getStartOffset()); + return; + } + try { + final int batchSize = config.getInt("sync.batch.size", 100000); + final ByteBuffer buffer = result.getBuffer(); + int size = result.getSize(); + if (size > batchSize) { + buffer.limit(batchSize); + size = batchSize; + } + final RemotingHeader header = RemotingBuilder.buildResponseHeader(CommandCode.SUCCESS, entry.getRequestHeader()); + ByteBuffer headerBuffer = HeaderSerializer.serialize(header, SYNC_HEADER_LEN + size, SYNC_HEADER_LEN); + headerBuffer.putInt(size); + headerBuffer.putLong(result.getStartOffset()); + headerBuffer.flip(); + entry.getCtx().writeAndFlush(new DataTransfer(headerBuffer, result, size)); + } catch (Exception e) { + result.release(); + } + } + + private boolean acquireSegmentLock(final SegmentBuffer result) { + if (BrokerConfig.getBrokerRole() != BrokerRole.MASTER) { + return true; + } + + final LogSegment segment = result.getLogSegment(); + if (segment == null) { + return true; + } + + if (currentSegment != null && currentSegment.getBaseOffset() == segment.getBaseOffset()) { + return true; + } + + if (currentSegment != null) { + final boolean release = currentSegment.release(); + LOG.info("release segment lock for {} while sync log, release result: {}", currentSegment, release); + currentSegment = null; + } + + final boolean retain = segment.retain(); + LOG.info("acquire segment lock for {} while sync log, retain result: {}", segment, retain); + if (retain) { + currentSegment = segment; + } + return retain; + } + + private static class SyncLogPayloadHolder implements PayloadHolder { + private final long startOffset; + + SyncLogPayloadHolder(long startOffset) { + this.startOffset = startOffset; + } + + @Override + public void writeBody(ByteBuf out) { + out.writeInt(0); + out.writeLong(startOffset); + } + } + + static class SyncRequestTimeoutTask implements TimerTask { + private final SyncRequestEntry entry; + private final SyncProcessor processor; + + SyncRequestTimeoutTask(SyncRequestEntry entry, SyncProcessor processor) { + this.entry = entry; + this.processor = processor; + } + + @Override + public void run(Timeout timeout) { + try { + processor.processTimeout(entry); + } catch (Exception e) { + LOG.error("process sync request error", e); + } + } + } +} diff --git a/qmq-server/src/main/java/qunar/tc/qmq/sync/master/ActionLogSyncWorker.java b/qmq-server/src/main/java/qunar/tc/qmq/sync/master/ActionLogSyncWorker.java new file mode 100644 index 00000000..c1c87e76 --- /dev/null +++ b/qmq-server/src/main/java/qunar/tc/qmq/sync/master/ActionLogSyncWorker.java @@ -0,0 +1,50 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.sync.master; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import qunar.tc.qmq.base.SyncRequest; +import qunar.tc.qmq.configuration.DynamicConfig; +import qunar.tc.qmq.store.Storage; +import qunar.tc.qmq.store.SegmentBuffer; + +/** + * @author yunfeng.yang + * @since 2017/8/19 + */ +class ActionLogSyncWorker extends AbstractLogSyncWorker { + private static final Logger LOG = LoggerFactory.getLogger(ActionLogSyncWorker.class); + + private final Storage storage; + + ActionLogSyncWorker(Storage storage, DynamicConfig config) { + super(config); + this.storage = storage; + } + + @Override + protected SegmentBuffer getSyncLog(SyncRequest syncRequest) { + long startSyncOffset = syncRequest.getActionLogOffset(); + long minActionLogOffset = storage.getMinActionLogOffset(); + if (startSyncOffset < minActionLogOffset) { + LOG.info("reset action log sync offset from {} to {}", startSyncOffset, minActionLogOffset); + startSyncOffset = minActionLogOffset; + } + return storage.getActionLogData(startSyncOffset); + } +} diff --git a/qmq-server/src/main/java/qunar/tc/qmq/sync/master/HeartbeatSyncWorker.java b/qmq-server/src/main/java/qunar/tc/qmq/sync/master/HeartbeatSyncWorker.java new file mode 100644 index 00000000..f612010c --- /dev/null +++ b/qmq-server/src/main/java/qunar/tc/qmq/sync/master/HeartbeatSyncWorker.java @@ -0,0 +1,78 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.sync.master; + +import io.netty.buffer.ByteBuf; +import io.netty.channel.ChannelHandlerContext; +import qunar.tc.qmq.configuration.BrokerConfig; +import qunar.tc.qmq.monitor.QMon; +import qunar.tc.qmq.protocol.CommandCode; +import qunar.tc.qmq.protocol.Datagram; +import qunar.tc.qmq.protocol.PayloadHolder; +import qunar.tc.qmq.store.Storage; +import qunar.tc.qmq.util.RemotingBuilder; + +/** + * @author yunfeng.yang + * @since 2017/8/20 + */ +class HeartbeatSyncWorker implements SyncProcessor { + private final Storage storage; + private volatile long slaveMessageLogLag = 0; + private volatile long slaveActionLogLag = 0; + + HeartbeatSyncWorker(Storage storage) { + this.storage = storage; + final String role = BrokerConfig.getBrokerRole().toString(); + QMon.slaveMessageLogLagGauge(role, () -> (double) slaveMessageLogLag); + QMon.slaveActionLogLagGauge(role, () -> (double) slaveActionLogLag); + } + + @Override + public void process(SyncRequestEntry requestEntry) { + final ChannelHandlerContext ctx = requestEntry.getCtx(); + final long messageLogMaxOffset = storage.getMaxMessageOffset(); + final long actionLogMaxOffset = storage.getMaxActionLogOffset(); + slaveMessageLogLag = messageLogMaxOffset - requestEntry.getSyncRequest().getMessageLogOffset(); + slaveActionLogLag = actionLogMaxOffset - requestEntry.getSyncRequest().getActionLogOffset(); + + final HeartbeatPayloadHolder payloadHolder = new HeartbeatPayloadHolder(messageLogMaxOffset, actionLogMaxOffset); + final Datagram datagram = RemotingBuilder.buildResponseDatagram(CommandCode.SUCCESS, requestEntry.getRequestHeader(), payloadHolder); + ctx.writeAndFlush(datagram); + } + + @Override + public void processTimeout(SyncRequestEntry entry) { + // do nothing + } + + private static class HeartbeatPayloadHolder implements PayloadHolder { + private final long messageLogMaxOffset; + private final long actionLogMaxOffset; + + HeartbeatPayloadHolder(long messageLogMaxOffset, long actionLogMaxOffset) { + this.messageLogMaxOffset = messageLogMaxOffset; + this.actionLogMaxOffset = actionLogMaxOffset; + } + + @Override + public void writeBody(ByteBuf out) { + out.writeLong(messageLogMaxOffset); + out.writeLong(actionLogMaxOffset); + } + } +} diff --git a/qmq-server/src/main/java/qunar/tc/qmq/sync/master/MasterSyncNettyServer.java b/qmq-server/src/main/java/qunar/tc/qmq/sync/master/MasterSyncNettyServer.java new file mode 100644 index 00000000..fa61278a --- /dev/null +++ b/qmq-server/src/main/java/qunar/tc/qmq/sync/master/MasterSyncNettyServer.java @@ -0,0 +1,56 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.sync.master; + +import qunar.tc.qmq.common.Disposable; +import qunar.tc.qmq.configuration.DynamicConfig; +import qunar.tc.qmq.netty.DefaultConnectionEventHandler; +import qunar.tc.qmq.netty.NettyServer; +import qunar.tc.qmq.protocol.CommandCode; +import qunar.tc.qmq.store.Storage; + +/** + * @author yunfeng.yang + * @since 2017/8/19 + */ +public class MasterSyncNettyServer implements Disposable { + private final NettyServer nettyServer; + private final SyncLogProcessor syncLogProcessor; + private final SyncCheckpointProcessor syncCheckpointProcessor; + + public MasterSyncNettyServer(final DynamicConfig config, final Storage storage) { + final Integer port = config.getInt("sync.port", 20882); + this.nettyServer = new NettyServer("sync", 4, port, new DefaultConnectionEventHandler("sync")); + this.syncLogProcessor = new SyncLogProcessor(storage, config); + this.syncCheckpointProcessor = new SyncCheckpointProcessor(storage); + } + + public void registerSyncEvent(Object listener) { + syncLogProcessor.registerSyncEvent(listener); + } + + public void start() { + nettyServer.registerProcessor(CommandCode.SYNC_LOG_REQUEST, syncLogProcessor); + nettyServer.registerProcessor(CommandCode.SYNC_CHECKPOINT_REQUEST, syncCheckpointProcessor); + nettyServer.start(); + } + + @Override + public void destroy() { + nettyServer.destroy(); + } +} diff --git a/qmq-server/src/main/java/qunar/tc/qmq/sync/master/MessageLogSyncWorker.java b/qmq-server/src/main/java/qunar/tc/qmq/sync/master/MessageLogSyncWorker.java new file mode 100644 index 00000000..34bf612a --- /dev/null +++ b/qmq-server/src/main/java/qunar/tc/qmq/sync/master/MessageLogSyncWorker.java @@ -0,0 +1,72 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.sync.master; + +import com.google.common.eventbus.AsyncEventBus; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import qunar.tc.qmq.base.SyncRequest; +import qunar.tc.qmq.common.Disposable; +import qunar.tc.qmq.concurrent.NamedThreadFactory; +import qunar.tc.qmq.configuration.DynamicConfig; +import qunar.tc.qmq.store.Storage; +import qunar.tc.qmq.store.SegmentBuffer; + +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; + +/** + * @author yunfeng.yang + * @since 2017/8/19 + */ +class MessageLogSyncWorker extends AbstractLogSyncWorker implements Disposable { + private static final Logger LOG = LoggerFactory.getLogger(MessageLogSyncWorker.class); + + private final Storage storage; + private final AsyncEventBus messageLogSyncEventBus; + private final ExecutorService dispatchExecutor; + + MessageLogSyncWorker(Storage storage, DynamicConfig config) { + super(config); + this.storage = storage; + this.dispatchExecutor = Executors.newSingleThreadExecutor(new NamedThreadFactory("heart-event-bus")); + this.messageLogSyncEventBus = new AsyncEventBus(dispatchExecutor); + } + + @Override + protected SegmentBuffer getSyncLog(SyncRequest syncRequest) { + messageLogSyncEventBus.post(syncRequest); + long startSyncOffset = syncRequest.getMessageLogOffset(); + long minMessageOffset = storage.getMinMessageOffset(); + if (startSyncOffset < minMessageOffset) { + LOG.info("reset message log sync offset from {} to {}", startSyncOffset, minMessageOffset); + startSyncOffset = minMessageOffset; + } + return storage.getMessageData(startSyncOffset); + } + + void registerSyncEvent(Object listener) { + messageLogSyncEventBus.register(listener); + } + + @Override + public void destroy() { + if (dispatchExecutor != null) { + dispatchExecutor.shutdown(); + } + } +} diff --git a/qmq-server/src/main/java/qunar/tc/qmq/sync/master/SyncCheckpointProcessor.java b/qmq-server/src/main/java/qunar/tc/qmq/sync/master/SyncCheckpointProcessor.java new file mode 100644 index 00000000..695c1c7f --- /dev/null +++ b/qmq-server/src/main/java/qunar/tc/qmq/sync/master/SyncCheckpointProcessor.java @@ -0,0 +1,77 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.sync.master; + +import io.netty.buffer.ByteBuf; +import io.netty.channel.ChannelHandlerContext; +import qunar.tc.qmq.netty.NettyRequestProcessor; +import qunar.tc.qmq.protocol.CommandCode; +import qunar.tc.qmq.protocol.Datagram; +import qunar.tc.qmq.protocol.PayloadHolder; +import qunar.tc.qmq.protocol.RemotingCommand; +import qunar.tc.qmq.store.CheckpointManager; +import qunar.tc.qmq.store.Storage; +import qunar.tc.qmq.util.RemotingBuilder; + +import java.util.concurrent.CompletableFuture; + +/** + * @author keli.wang + * @since 2018/10/29 + */ +class SyncCheckpointProcessor implements NettyRequestProcessor { + private final CheckpointManager checkpointManager; + + SyncCheckpointProcessor(Storage storage) { + this.checkpointManager = storage.getCheckpointManager(); + } + + @Override + public CompletableFuture processRequest(ChannelHandlerContext ctx, RemotingCommand request) { + final byte[] message = checkpointManager.dumpMessageCheckpoint(); + final byte[] action = checkpointManager.dumpActionCheckpoint(); + final CheckpointPayloadHolder payloadHolder = new CheckpointPayloadHolder(message, action); + final Datagram datagram = RemotingBuilder.buildResponseDatagram(CommandCode.SUCCESS, request.getHeader(), payloadHolder); + return CompletableFuture.completedFuture(datagram); + } + + @Override + public boolean rejectRequest() { + return false; + } + + private static class CheckpointPayloadHolder implements PayloadHolder { + private static final int V1 = 1; + + private final byte[] message; + private final byte[] action; + + CheckpointPayloadHolder(final byte[] message, final byte[] action) { + this.message = message; + this.action = action; + } + + @Override + public void writeBody(ByteBuf out) { + out.writeByte(V1); + out.writeInt(message.length); + out.writeBytes(message); + out.writeInt(action.length); + out.writeBytes(action); + } + } +} diff --git a/qmq-server/src/main/java/qunar/tc/qmq/sync/master/SyncLogProcessor.java b/qmq-server/src/main/java/qunar/tc/qmq/sync/master/SyncLogProcessor.java new file mode 100644 index 00000000..72e32b43 --- /dev/null +++ b/qmq-server/src/main/java/qunar/tc/qmq/sync/master/SyncLogProcessor.java @@ -0,0 +1,120 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.sync.master; + +import io.netty.buffer.ByteBuf; +import io.netty.channel.ChannelHandlerContext; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import qunar.tc.qmq.base.SyncRequest; +import qunar.tc.qmq.common.Disposable; +import qunar.tc.qmq.concurrent.NamedThreadFactory; +import qunar.tc.qmq.configuration.DynamicConfig; +import qunar.tc.qmq.netty.NettyRequestProcessor; +import qunar.tc.qmq.protocol.CommandCode; +import qunar.tc.qmq.protocol.Datagram; +import qunar.tc.qmq.protocol.RemotingCommand; +import qunar.tc.qmq.store.Storage; +import qunar.tc.qmq.sync.SyncType; +import qunar.tc.qmq.util.RemotingBuilder; + +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; + +/** + * @author yunfeng.yang + * @since 2017/8/19 + */ +class SyncLogProcessor implements NettyRequestProcessor, Disposable { + private static final Logger LOG = LoggerFactory.getLogger(SyncLogProcessor.class); + + private final ExecutorService executor; + private final MessageLogSyncWorker messageLogSyncWorker; + private final Map processorMap; + + SyncLogProcessor(Storage storage, DynamicConfig config) { + this.executor = Executors.newFixedThreadPool(3, new NamedThreadFactory("master-sync")); + this.processorMap = new HashMap<>(); + this.messageLogSyncWorker = new MessageLogSyncWorker(storage, config); + final ActionLogSyncWorker actionLogSyncWorker = new ActionLogSyncWorker(storage, config); + final HeartbeatSyncWorker heartBeatSyncWorker = new HeartbeatSyncWorker(storage); + processorMap.put(SyncType.message.getCode(), messageLogSyncWorker); + processorMap.put(SyncType.action.getCode(), actionLogSyncWorker); + processorMap.put(SyncType.heartbeat.getCode(), heartBeatSyncWorker); + } + + @Override + public CompletableFuture processRequest(ChannelHandlerContext ctx, RemotingCommand request) { + final SyncRequest syncRequest = deserializeSyncRequest(request); + final int syncType = syncRequest.getSyncType(); + final SyncProcessor processor = processorMap.get(syncType); + if (processor == null) { + LOG.error("unknown sync type {}", syncType); + final Datagram datagram = RemotingBuilder.buildEmptyResponseDatagram(CommandCode.BROKER_ERROR, request.getHeader()); + return CompletableFuture.completedFuture(datagram); + } + + final SyncRequestEntry entry = new SyncRequestEntry(ctx, request.getHeader(), syncRequest); + executor.submit(new SyncRequestProcessTask(entry, processor)); + return null; + } + + @Override + public boolean rejectRequest() { + return false; + } + + void registerSyncEvent(Object listener) { + this.messageLogSyncWorker.registerSyncEvent(listener); + } + + private SyncRequest deserializeSyncRequest(final RemotingCommand request) { + final ByteBuf body = request.getBody(); + final int logType = body.readByte(); + final long messageLogOffset = body.readLong(); + final long actionLogOffset = body.readLong(); + return new SyncRequest(logType, messageLogOffset, actionLogOffset); + } + + @Override + public void destroy() { + messageLogSyncWorker.destroy(); + executor.shutdown(); + } + + private static class SyncRequestProcessTask implements Runnable { + private final SyncRequestEntry entry; + private final SyncProcessor processor; + + private SyncRequestProcessTask(SyncRequestEntry entry, SyncProcessor processor) { + this.entry = entry; + this.processor = processor; + } + + @Override + public void run() { + try { + processor.process(entry); + } catch (Exception e) { + LOG.error("process sync request error", e); + } + } + } +} diff --git a/qmq-server/src/main/java/qunar/tc/qmq/sync/master/SyncProcessor.java b/qmq-server/src/main/java/qunar/tc/qmq/sync/master/SyncProcessor.java new file mode 100644 index 00000000..abc7106c --- /dev/null +++ b/qmq-server/src/main/java/qunar/tc/qmq/sync/master/SyncProcessor.java @@ -0,0 +1,27 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.sync.master; + +/** + * @author keli.wang + * @since 2018/11/2 + */ +interface SyncProcessor { + void process(SyncRequestEntry entry); + + void processTimeout(SyncRequestEntry entry); +} diff --git a/qmq-server/src/main/java/qunar/tc/qmq/sync/master/SyncRequestEntry.java b/qmq-server/src/main/java/qunar/tc/qmq/sync/master/SyncRequestEntry.java new file mode 100644 index 00000000..01e872cd --- /dev/null +++ b/qmq-server/src/main/java/qunar/tc/qmq/sync/master/SyncRequestEntry.java @@ -0,0 +1,49 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.sync.master; + +import io.netty.channel.ChannelHandlerContext; +import qunar.tc.qmq.base.SyncRequest; +import qunar.tc.qmq.protocol.RemotingHeader; + +/** + * @author keli.wang + * @since 2018/11/2 + */ +class SyncRequestEntry { + private final ChannelHandlerContext ctx; + private final RemotingHeader requestHeader; + private final SyncRequest syncRequest; + + SyncRequestEntry(ChannelHandlerContext ctx, RemotingHeader requestHeader, SyncRequest syncRequest) { + this.ctx = ctx; + this.requestHeader = requestHeader; + this.syncRequest = syncRequest; + } + + ChannelHandlerContext getCtx() { + return ctx; + } + + RemotingHeader getRequestHeader() { + return requestHeader; + } + + SyncRequest getSyncRequest() { + return syncRequest; + } +} diff --git a/qmq-server/src/main/java/qunar/tc/qmq/utils/ConsumerGroupUtils.java b/qmq-server/src/main/java/qunar/tc/qmq/utils/ConsumerGroupUtils.java new file mode 100644 index 00000000..7af64a3c --- /dev/null +++ b/qmq-server/src/main/java/qunar/tc/qmq/utils/ConsumerGroupUtils.java @@ -0,0 +1,30 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.utils; + +/** + * @author yunfeng.yang + * @since 2017/7/31 + */ +public final class ConsumerGroupUtils { + private ConsumerGroupUtils() { + } + + public static String buildConsumerGroupKey(String subject, String group) { + return subject + "-" + group; + } +} diff --git a/qmq-server/src/main/java/qunar/tc/qmq/utils/HeaderSerializer.java b/qmq-server/src/main/java/qunar/tc/qmq/utils/HeaderSerializer.java new file mode 100644 index 00000000..089d3fca --- /dev/null +++ b/qmq-server/src/main/java/qunar/tc/qmq/utils/HeaderSerializer.java @@ -0,0 +1,54 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.utils; + +import qunar.tc.qmq.protocol.RemotingHeader; + +import java.nio.ByteBuffer; + +import static qunar.tc.qmq.protocol.RemotingHeader.*; + +public class HeaderSerializer { + public static ByteBuffer serialize(RemotingHeader header, int payloadSize, int additional) { + short headerLength = MIN_HEADER_SIZE; + if (header.getVersion() >= RemotingHeader.VERSION_3) { + headerLength += REQUEST_CODE_LEN; + } + + int bufferLength = TOTAL_SIZE_LEN + HEADER_SIZE_LEN + headerLength + additional; + ByteBuffer buffer = ByteBuffer.allocate(bufferLength); + // total len + int total = HEADER_SIZE_LEN + headerLength + payloadSize; + buffer.putInt(total); + // header len + buffer.putShort(headerLength); + // magic code + buffer.putInt(header.getMagicCode()); + // code + buffer.putShort(header.getCode()); + // version + buffer.putShort(header.getVersion()); + // opaque + buffer.putInt(header.getOpaque()); + // flag + buffer.putInt(header.getFlag()); + if (header.getVersion() >= RemotingHeader.VERSION_3) { + buffer.putShort(header.getRequestCode()); + } + return buffer; + } +} diff --git a/qmq-server/src/main/java/qunar/tc/qmq/utils/ServerTimerUtil.java b/qmq-server/src/main/java/qunar/tc/qmq/utils/ServerTimerUtil.java new file mode 100644 index 00000000..7c828d96 --- /dev/null +++ b/qmq-server/src/main/java/qunar/tc/qmq/utils/ServerTimerUtil.java @@ -0,0 +1,37 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.utils; + +import io.netty.util.HashedWheelTimer; +import io.netty.util.Timeout; +import io.netty.util.TimerTask; +import qunar.tc.qmq.concurrent.NamedThreadFactory; + +import java.util.concurrent.TimeUnit; + +/** + * @author keli.wang + * @since 2018/11/2 + */ +public class ServerTimerUtil { + private static final HashedWheelTimer TIMER = new HashedWheelTimer(new NamedThreadFactory("qmq-server-timer"), 1, TimeUnit.MILLISECONDS); + + + public static Timeout newTimeout(final TimerTask task, final long delay, final TimeUnit unit) { + return TIMER.newTimeout(task, delay, unit); + } +} diff --git a/qmq-store/pom.xml b/qmq-store/pom.xml new file mode 100644 index 00000000..4d47259f --- /dev/null +++ b/qmq-store/pom.xml @@ -0,0 +1,65 @@ + + + 4.0.0 + + + qmq + qunar.tc + 4.0.30 + + + qmq-store + + + + ${project.groupId} + qmq-common + + + ${project.groupId} + qmq-server-common + + + + com.google.guava + guava + + + org.slf4j + slf4j-api + + + io.netty + netty-all + + + com.fasterxml.jackson.core + jackson-core + + + com.fasterxml.jackson.core + jackson-databind + + + + ch.qos.logback + logback-classic + test + + + + junit + junit + + + + + + + src/main/resources + + + + \ No newline at end of file diff --git a/qmq-store/src/main/c/MmapUtil.c b/qmq-store/src/main/c/MmapUtil.c new file mode 100644 index 00000000..703549b5 --- /dev/null +++ b/qmq-store/src/main/c/MmapUtil.c @@ -0,0 +1,15 @@ +#include "MmapUtil.h" +#include "jni.h" +#include +#include +#include + +JNIEXPORT void JNICALL Java_qunar_tc_qmq_store_MmapUtil_free0(JNIEnv * env, jclass clazz, jint fd, jlong offset, jlong len) +{ + int result = posix_fadvise((int)fd, (off_t)offset, (off_t)len, POSIX_FADV_DONTNEED); + if (result == -1) { + jclass cls = (*env)->FindClass(env, "java/io/IOException"); + if(cls != 0) + (*env)->ThrowNew(env, cls, "madvise error"); + } +} \ No newline at end of file diff --git a/qmq-store/src/main/c/MmapUtil.h b/qmq-store/src/main/c/MmapUtil.h new file mode 100644 index 00000000..7d4ea642 --- /dev/null +++ b/qmq-store/src/main/c/MmapUtil.h @@ -0,0 +1,20 @@ +/* DO NOT EDIT THIS FILE - it is machine generated */ +#include +/* Header for class qunar_tc_qmq_store_MmapUtil */ + +#ifndef _Included_qunar_tc_qmq_store_MmapUtil +#define _Included_qunar_tc_qmq_store_MmapUtil +#ifdef __cplusplus +extern "C" { +#endif +/* + * Class: qunar_tc_qmq_store_MmapUtil + * Method: free0 + * Signature: (JJ)V + */ +JNIEXPORT void JNICALL Java_qunar_tc_qmq_store_MmapUtil_free0(JNIEnv *, jclass, jint, jlong, jlong); + +#ifdef __cplusplus +} +#endif +#endif \ No newline at end of file diff --git a/qmq-store/src/main/java/qunar/tc/qmq/store/Action.java b/qmq-store/src/main/java/qunar/tc/qmq/store/Action.java new file mode 100644 index 00000000..74613847 --- /dev/null +++ b/qmq-store/src/main/java/qunar/tc/qmq/store/Action.java @@ -0,0 +1,33 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.store; + +/** + * @author keli.wang + * @since 2017/8/20 + */ +public interface Action { + ActionType type(); + + String subject(); + + String group(); + + String consumerId(); + + long timestamp(); +} diff --git a/qmq-store/src/main/java/qunar/tc/qmq/store/ActionCheckpoint.java b/qmq-store/src/main/java/qunar/tc/qmq/store/ActionCheckpoint.java new file mode 100644 index 00000000..36ae26eb --- /dev/null +++ b/qmq-store/src/main/java/qunar/tc/qmq/store/ActionCheckpoint.java @@ -0,0 +1,57 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.store; + +import com.google.common.collect.Table; + +/** + * @author keli.wang + * @since 2018/9/10 + */ +public class ActionCheckpoint { + // mark this object is read from old version snapshot file + // TODO(keli.wang): delete this after all broker group is migrate to new snapshot format + private final boolean fromOldVersion; + private final Table progresses; + private long offset; + + public ActionCheckpoint(long offset, Table progresses) { + this(false, offset, progresses); + } + + public ActionCheckpoint(boolean fromOldVersion, long offset, Table progresses) { + this.fromOldVersion = fromOldVersion; + this.offset = offset; + this.progresses = progresses; + } + + public boolean isFromOldVersion() { + return fromOldVersion; + } + + public long getOffset() { + return offset; + } + + public void setOffset(long offset) { + this.offset = offset; + } + + public Table getProgresses() { + return progresses; + } +} diff --git a/qmq-store/src/main/java/qunar/tc/qmq/store/ActionCheckpointSerde.java b/qmq-store/src/main/java/qunar/tc/qmq/store/ActionCheckpointSerde.java new file mode 100644 index 00000000..421fa5cd --- /dev/null +++ b/qmq-store/src/main/java/qunar/tc/qmq/store/ActionCheckpointSerde.java @@ -0,0 +1,188 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.store; + +import com.google.common.base.Charsets; +import com.google.common.base.Joiner; +import com.google.common.base.Splitter; +import com.google.common.base.Strings; +import com.google.common.collect.HashBasedTable; +import com.google.common.collect.Table; +import com.google.common.io.LineReader; + +import java.io.IOException; +import java.io.StringReader; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * @author keli.wang + * @since 2018/9/10 + */ +public class ActionCheckpointSerde implements Serde { + private static final int VERSION_V1 = 1; + private static final int VERSION_V2 = 2; + private static final int VERSION_V3 = 3; + + private static final char NEWLINE = '\n'; + private static final Joiner SLASH_JOINER = Joiner.on('/'); + private static final Splitter SLASH_SPLITTER = Splitter.on('/'); + private static final Splitter COMMA_SPLITTER = Splitter.on(','); + + @Override + public byte[] toBytes(final ActionCheckpoint state) { + final StringBuilder data = new StringBuilder(); + data.append(VERSION_V3).append(NEWLINE); + data.append(state.getOffset()).append(NEWLINE); + + final Table progresses = state.getProgresses(); + for (final String subject : progresses.rowKeySet()) { + final Map groups = progresses.row(subject); + data.append(SLASH_JOINER.join(subject, groups.size())).append(NEWLINE); + + for (final String group : groups.keySet()) { + final ConsumerGroupProgress progress = groups.get(group); + final Map consumers = progress.getConsumers(); + final int consumerCount = consumers == null ? 0 : consumers.size(); + + data.append(SLASH_JOINER.join(group, boolean2Short(progress.isBroadcast()), progress.getPull(), consumerCount)).append(NEWLINE); + + if (consumerCount <= 0) { + continue; + } + + consumers.values().forEach(consumer -> { + data.append(SLASH_JOINER.join(consumer.getConsumerId(), consumer.getPull(), consumer.getAck())).append(NEWLINE); + }); + } + } + return data.toString().getBytes(Charsets.UTF_8); + } + + @Override + public ActionCheckpoint fromBytes(final byte[] data) { + + try { + final LineReader reader = new LineReader(new StringReader(new String(data, Charsets.UTF_8))); + final int version = Integer.parseInt(reader.readLine()); + switch (version) { + case VERSION_V1: + return parseBySplitter(reader, COMMA_SPLITTER); + case VERSION_V2: + return parseBySplitter(reader, SLASH_SPLITTER); + case VERSION_V3: + return parseV3(reader); + default: + throw new RuntimeException("unknown snapshot content version " + version); + + } + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + + private ActionCheckpoint parseBySplitter(final LineReader reader, final Splitter splitter) throws IOException { + final Table progresses = HashBasedTable.create(); + + while (true) { + final String subjectLine = reader.readLine(); + if (Strings.isNullOrEmpty(subjectLine)) { + break; + } + + final List subjectParts = splitter.splitToList(subjectLine); + final String subject = subjectParts.get(0); + final int groupCount = Integer.parseInt(subjectParts.get(1)); + for (int i = 0; i < groupCount; i++) { + final String groupLine = reader.readLine(); + final List groupParts = splitter.splitToList(groupLine); + final String group = groupParts.get(0); + final long maxPulledMessageSequence = Long.parseLong(groupParts.get(1)); + final int consumerCount = Integer.parseInt(groupParts.get(2)); + + final ConsumerGroupProgress progress = new ConsumerGroupProgress(subject, group, false, maxPulledMessageSequence, new HashMap<>(consumerCount)); + progresses.put(subject, group, progress); + + final Map consumers = progress.getConsumers(); + for (int j = 0; j < consumerCount; j++) { + final String consumerLine = reader.readLine(); + final List consumerParts = splitter.splitToList(consumerLine); + final String consumerId = consumerParts.get(0); + final long maxAckedPullLogSequence = Long.parseLong(consumerParts.get(1)); + + consumers.put(consumerId, new ConsumerProgress(subject, group, consumerId, -1, maxAckedPullLogSequence)); + } + } + } + + return new ActionCheckpoint(true, -1, progresses); + } + + private ActionCheckpoint parseV3(LineReader reader) throws IOException { + final long offset = Long.parseLong(reader.readLine()); + + final Table progresses = HashBasedTable.create(); + while (true) { + final String subjectLine = reader.readLine(); + if (Strings.isNullOrEmpty(subjectLine)) { + break; + } + + final List subjectParts = SLASH_SPLITTER.splitToList(subjectLine); + final String subject = subjectParts.get(0); + final int groupCount = Integer.parseInt(subjectParts.get(1)); + for (int i = 0; i < groupCount; i++) { + final String groupLine = reader.readLine(); + final List groupParts = SLASH_SPLITTER.splitToList(groupLine); + final String group = groupParts.get(0); + final boolean broadcast = short2Boolean(Short.parseShort(groupParts.get(1))); + final long maxPulledMessageSequence = Long.parseLong(groupParts.get(2)); + final int consumerCount = Integer.parseInt(groupParts.get(3)); + + final ConsumerGroupProgress progress = new ConsumerGroupProgress(subject, group, broadcast, maxPulledMessageSequence, new HashMap<>(consumerCount)); + progresses.put(subject, group, progress); + + final Map consumers = progress.getConsumers(); + for (int j = 0; j < consumerCount; j++) { + final String consumerLine = reader.readLine(); + final List consumerParts = SLASH_SPLITTER.splitToList(consumerLine); + final String consumerId = consumerParts.get(0); + final long pull = Long.parseLong(consumerParts.get(1)); + final long ack = Long.parseLong(consumerParts.get(2)); + + consumers.put(consumerId, new ConsumerProgress(subject, group, consumerId, pull, ack)); + } + } + } + + return new ActionCheckpoint(offset, progresses); + } + + private short boolean2Short(final boolean bool) { + if (bool) { + return 1; + } else { + return 0; + } + } + + private boolean short2Boolean(final short n) { + return n != 0; + } +} diff --git a/qmq-store/src/main/java/qunar/tc/qmq/store/ActionLog.java b/qmq-store/src/main/java/qunar/tc/qmq/store/ActionLog.java new file mode 100644 index 00000000..a4c092e9 --- /dev/null +++ b/qmq-store/src/main/java/qunar/tc/qmq/store/ActionLog.java @@ -0,0 +1,231 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.store; + +import com.google.common.base.Strings; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import qunar.tc.qmq.monitor.QMon; + +import java.io.File; +import java.nio.ByteBuffer; + +/** + * @author keli.wang + * @since 2017/8/20 + */ +public class ActionLog { + public static final int PER_SEGMENT_FILE_SIZE = 100 * 1024 * 1024; + private static final Logger LOG = LoggerFactory.getLogger(ActionLog.class); + private final StorageConfig config; + private final LogManager logManager; + private final MessageAppender actionAppender = new ActionAppender(); + + public ActionLog(final StorageConfig config) { + this.config = config; + this.logManager = new LogManager(new File(config.getActionLogStorePath()), PER_SEGMENT_FILE_SIZE, config, new ActionLogSegmentValidator(), true); + } + + public synchronized PutMessageResult addAction(final Action action) { + final AppendMessageResult result; + LogSegment segment = logManager.latestSegment(); + if (segment == null) { + segment = logManager.allocNextSegment(); + } + + if (segment == null) { + return new PutMessageResult(PutMessageStatus.CREATE_MAPPED_FILE_FAILED, null); + } + + result = segment.append(action, actionAppender); + switch (result.getStatus()) { + case SUCCESS: + break; + case END_OF_FILE: + if (logManager.allocNextSegment() == null) { + return new PutMessageResult(PutMessageStatus.CREATE_MAPPED_FILE_FAILED, null); + } + return addAction(action); + case MESSAGE_SIZE_EXCEEDED: + return new PutMessageResult(PutMessageStatus.MESSAGE_ILLEGAL, result); + default: + return new PutMessageResult(PutMessageStatus.UNKNOWN_ERROR, result); + } + + return new PutMessageResult(PutMessageStatus.SUCCESS, result); + } + + public boolean appendData(final long startOffset, final ByteBuffer data) { + LogSegment segment = logManager.locateSegment(startOffset); + if (segment == null) { + segment = logManager.allocOrResetSegments(startOffset); + fillPreBlank(segment, startOffset); + } + + return segment.appendData(data); + } + + private void fillPreBlank(LogSegment segment, long untilWhere) { + final ByteBuffer buffer = ByteBuffer.allocate(9); + buffer.putInt(MagicCode.ACTION_LOG_MAGIC_V1); + buffer.put((byte) 2); + buffer.putInt((int) (untilWhere % PER_SEGMENT_FILE_SIZE)); + segment.fillPreBlank(buffer, untilWhere); + } + + public SegmentBuffer getMessageData(final long offset) { + final LogSegment segment = logManager.locateSegment(offset); + if (segment == null) { + return null; + } + + final int pos = (int) (offset % PER_SEGMENT_FILE_SIZE); + return segment.selectSegmentBuffer(pos); + } + + public ActionLogVisitor newVisitor(final long start) { + return new ActionLogVisitor(logManager, start); + } + + public long getMaxOffset() { + return logManager.getMaxOffset(); + } + + public long getMinOffset() { + return logManager.getMinOffset(); + } + + public void flush() { + final long start = System.currentTimeMillis(); + try { + logManager.flush(); + } finally { + QMon.flushActionLogTimer(System.currentTimeMillis() - start); + } + } + + public void close() { + logManager.close(); + } + + public void clean() { + logManager.deleteExpiredSegments(config.getLogRetentionMs()); + } + + private class ActionAppender implements MessageAppender { + private static final int MIN_RECORD_BYTES = 5; // 4 bytes magic + 1 byte record type + private static final int MAX_BYTES = 1024 * 1024 * 10; // 10M + + private final ByteBuffer workingBuffer = ByteBuffer.allocate(MAX_BYTES); + + @Override + public AppendMessageResult doAppend(long baseOffset, ByteBuffer targetBuffer, int freeSpace, Action action) { + workingBuffer.clear(); + final int size = fillBuffer(workingBuffer, action); + final long wroteOffset = baseOffset + targetBuffer.position(); + + if (size != freeSpace && size + MIN_RECORD_BYTES > freeSpace) { + workingBuffer.clear(); + workingBuffer.limit(freeSpace); + workingBuffer.putInt(MagicCode.ACTION_LOG_MAGIC_V1); + workingBuffer.put((byte) 1); + targetBuffer.put(workingBuffer.array(), 0, freeSpace); + return new AppendMessageResult<>(AppendMessageStatus.END_OF_FILE, wroteOffset, freeSpace, null); + } else { + workingBuffer.limit(size); + targetBuffer.put(workingBuffer.array(), 0, size); + + return new AppendMessageResult<>(AppendMessageStatus.SUCCESS, wroteOffset, size, new MessageSequence(wroteOffset, wroteOffset)); + } + } + + private int fillBuffer(final ByteBuffer buffer, final Action action) { + final int startIndex = buffer.position(); + buffer.putInt(MagicCode.ACTION_LOG_MAGIC_V1); + buffer.put((byte) 0); + buffer.put(action.type().getCode()); + + final int payloadSizeIndex = buffer.position(); + buffer.position(buffer.position() + Integer.BYTES); + + // TODO(keli.wang): add monitor here + final int payloadSize = action.type().getReaderWriter().write(buffer, action); + + buffer.putInt(payloadSizeIndex, payloadSize); + + return buffer.position() - startIndex; + } + } + + private class ActionLogSegmentValidator implements LogSegmentValidator { + @Override + public ValidateResult validate(LogSegment segment) { + final int fileSize = segment.getFileSize(); + final ByteBuffer buffer = segment.sliceByteBuffer(); + + int position = 0; + while (true) { + if (position == fileSize) { + return new ValidateResult(ValidateStatus.COMPLETE, fileSize); + } + + final int result = consumeAndValidateMessage(buffer); + if (result == -1) { + return new ValidateResult(ValidateStatus.PARTIAL, position); + } else if (result == 0) { + return new ValidateResult(ValidateStatus.COMPLETE, fileSize); + } else { + position += result; + } + } + } + + private int consumeAndValidateMessage(final ByteBuffer buffer) { + final int magic = buffer.getInt(); + if (magic != MagicCode.ACTION_LOG_MAGIC_V1) { + return -1; + } + + final byte recordType = buffer.get(); + if (recordType == 2) { + return buffer.getInt(); + } else if (recordType == 1) { + return 0; + } else if (recordType == 0) { + try { + final ActionType payloadType = ActionType.fromCode(buffer.get()); + final int payloadSize = buffer.getInt(); + // TODO(keli.wang): 如果我们要校验action是否完全正确,还是得在readerwriter内部进行 + // 写完action之后得再写一位非0记录,这样才能完全判断记录不是partial的 + final Action action = payloadType.getReaderWriter().read(buffer); + if (Strings.isNullOrEmpty(action.subject()) + || Strings.isNullOrEmpty(action.group()) + || Strings.isNullOrEmpty(action.consumerId()) + || action.timestamp() <= 0) { + return -1; + } + return payloadSize + 10; + } catch (Exception e) { + LOG.error("fail read action log", e); + return -1; + } + } else { + return -1; + } + } + } +} diff --git a/qmq-store/src/main/java/qunar/tc/qmq/store/ActionLogIterateService.java b/qmq-store/src/main/java/qunar/tc/qmq/store/ActionLogIterateService.java new file mode 100644 index 00000000..cbeca283 --- /dev/null +++ b/qmq-store/src/main/java/qunar/tc/qmq/store/ActionLogIterateService.java @@ -0,0 +1,137 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.store; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import qunar.tc.qmq.monitor.QMon; +import qunar.tc.qmq.store.action.ActionEvent; +import qunar.tc.qmq.store.event.FixedExecOrderEventBus; + +import java.util.Optional; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.LongAdder; + +/** + * @author keli.wang + * @since 2017/8/21 + */ +public class ActionLogIterateService implements AutoCloseable { + private static final Logger LOG = LoggerFactory.getLogger(ActionLogIterateService.class); + + private final ActionLog log; + private final FixedExecOrderEventBus dispatcher; + private final Thread dispatcherThread; + + private final LongAdder iterateFrom; + private volatile boolean stop = false; + + public ActionLogIterateService(final ActionLog log, final CheckpointManager checkpointManager, final FixedExecOrderEventBus dispatcher) { + this.log = log; + this.dispatcher = dispatcher; + this.dispatcherThread = new Thread(new Dispatcher()); + this.dispatcherThread.setName("ActionLogIterator"); + this.iterateFrom = new LongAdder(); + this.iterateFrom.add(initialIterateFrom(log, checkpointManager)); + + QMon.replayActionLogLag(() -> (double) replayActionLogLag()); + } + + private long initialIterateFrom(final ActionLog log, final CheckpointManager checkpointManager) { + final long checkpointOffset = checkpointManager.getActionCheckpointOffset(); + final long maxOffset = log.getMaxOffset(); + + if (checkpointOffset <= 0) { + return maxOffset; + } + if (checkpointOffset > maxOffset) { + return maxOffset; + } + + return checkpointOffset; + } + + public void start() { + dispatcherThread.start(); + } + + public void blockUntilReplayDone() { + LOG.info("replay action log initial lag: {}; min: {}, max: {}, from: {}", + replayActionLogLag(), log.getMinOffset(), log.getMaxOffset(), iterateFrom.longValue()); + + while (replayActionLogLag() > 0) { + LOG.info("waiting replay action log ..."); + try { + TimeUnit.SECONDS.sleep(1); + } catch (InterruptedException e) { + LOG.warn("block until replay done interrupted", e); + } + } + } + + private long replayActionLogLag() { + return log.getMaxOffset() - iterateFrom.longValue(); + } + + @Override + public void close() { + stop = true; + try { + dispatcherThread.join(); + } catch (InterruptedException e) { + LOG.error("action log dispatcher thread interrupted", e); + } + } + + private class Dispatcher implements Runnable { + @Override + public void run() { + while (!stop) { + try { + processLog(); + } catch (Throwable e) { + QMon.replayActionLogFailedCountInc(); + LOG.error("replay action log failed, will retry.", e); + } + } + } + + private void processLog() { + long startOffset = iterateFrom.longValue(); + final ActionLogVisitor visitor = log.newVisitor(iterateFrom.longValue()); + if (startOffset != visitor.getStartOffset()) { + iterateFrom.reset(); + iterateFrom.add(visitor.getStartOffset()); + } + while (true) { + final Optional action = visitor.nextAction(); + if (action == null) { + break; + } + + action.ifPresent(act -> dispatcher.post(new ActionEvent(iterateFrom.longValue() + visitor.visitedBufferSize(), act))); + } + iterateFrom.add(visitor.visitedBufferSize()); + + try { + TimeUnit.MILLISECONDS.sleep(5); + } catch (InterruptedException e) { + LOG.warn("action log dispatcher sleep interrupted"); + } + } + } +} diff --git a/qmq-store/src/main/java/qunar/tc/qmq/store/ActionLogVisitor.java b/qmq-store/src/main/java/qunar/tc/qmq/store/ActionLogVisitor.java new file mode 100644 index 00000000..b9f910ca --- /dev/null +++ b/qmq-store/src/main/java/qunar/tc/qmq/store/ActionLogVisitor.java @@ -0,0 +1,128 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.store; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.nio.ByteBuffer; +import java.util.Optional; +import java.util.concurrent.atomic.AtomicInteger; + +/** + * @author keli.wang + * @since 2017/8/21 + */ +public class ActionLogVisitor { + private static final Logger LOG = LoggerFactory.getLogger(ActionLogVisitor.class); + + private static final int MIN_RECORD_BYTES = 5; // 4 bytes magic + 1 byte record type + + private final LogSegment currentSegment; + private final SegmentBuffer currentBuffer; + private final AtomicInteger visitedBufferSize = new AtomicInteger(0); + private final long startOffset; + + public ActionLogVisitor(final LogManager logManager, final long startOffset) { + final long initialOffset = initialOffset(logManager, startOffset); + this.currentSegment = logManager.locateSegment(initialOffset); + this.currentBuffer = selectBuffer(initialOffset); + this.startOffset = initialOffset; + } + + private long initialOffset(final LogManager logManager, final long originStart) { + if (originStart < logManager.getMinOffset()) { + LOG.error("initial action log visitor offset less than min offset. start: {}, min: {}", + originStart, logManager.getMinOffset()); + return logManager.getMinOffset(); + } + + return originStart; + } + + public Optional nextAction() { + if (currentBuffer == null) { + return null; + } + return readAction(currentBuffer.getBuffer()); + } + + public int visitedBufferSize() { + if (currentBuffer == null) { + return 0; + } + return visitedBufferSize.get(); + } + + private SegmentBuffer selectBuffer(final long startOffset) { + if (currentSegment == null) { + return null; + } + final int pos = (int) (startOffset % currentSegment.getFileSize()); + return currentSegment.selectSegmentBuffer(pos); + } + + // TODO(keli.wang): need replace optional with custom class + private Optional readAction(final ByteBuffer buffer) { + if (buffer.remaining() < MIN_RECORD_BYTES) { + return null; + } + + final int startPos = buffer.position(); + final int magic = buffer.getInt(); + if (magic != MagicCode.ACTION_LOG_MAGIC_V1) { + visitedBufferSize.set(currentBuffer.getSize()); + return null; + } + + final byte recordType = buffer.get(); + if (recordType == 2) { + if (buffer.remaining() < Integer.BYTES) { + return null; + } + final int blankSize = buffer.getInt(); + visitedBufferSize.addAndGet(blankSize + (buffer.position() - startPos)); + return Optional.empty(); + } else if (recordType == 1) { + visitedBufferSize.set(currentBuffer.getSize()); + return null; + } else if (recordType == 0) { + try { + if (buffer.remaining() < Integer.BYTES + Byte.BYTES) { + return null; + } + final ActionType payloadType = ActionType.fromCode(buffer.get()); + final int payloadSize = buffer.getInt(); + if (buffer.remaining() < payloadSize) { + return null; + } + final Action action = payloadType.getReaderWriter().read(buffer); + visitedBufferSize.addAndGet(buffer.position() - startPos); + return Optional.of(action); + } catch (Exception e) { + LOG.error("fail read action log", e); + return null; + } + } else { + throw new RuntimeException("Unknown record type"); + } + } + + public long getStartOffset() { + return startOffset; + } +} diff --git a/qmq-store/src/main/java/qunar/tc/qmq/store/ActionReaderWriter.java b/qmq-store/src/main/java/qunar/tc/qmq/store/ActionReaderWriter.java new file mode 100644 index 00000000..155f9142 --- /dev/null +++ b/qmq-store/src/main/java/qunar/tc/qmq/store/ActionReaderWriter.java @@ -0,0 +1,29 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.store; + +import java.nio.ByteBuffer; + +/** + * @author keli.wang + * @since 2017/8/20 + */ +public interface ActionReaderWriter { + int write(final ByteBuffer to, final Action action); + + Action read(final ByteBuffer from); +} diff --git a/qmq-store/src/main/java/qunar/tc/qmq/store/ActionType.java b/qmq-store/src/main/java/qunar/tc/qmq/store/ActionType.java new file mode 100644 index 00000000..9d1f2ea8 --- /dev/null +++ b/qmq-store/src/main/java/qunar/tc/qmq/store/ActionType.java @@ -0,0 +1,69 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.store; + +import com.google.common.collect.ImmutableMap; +import qunar.tc.qmq.store.action.ForeverOfflineActionReaderWriter; +import qunar.tc.qmq.store.action.PullActionReaderWriter; +import qunar.tc.qmq.store.action.RangeAckActionReaderWriter; + +import java.util.HashMap; +import java.util.Map; + +/** + * @author keli.wang + * @since 2017/8/20 + */ +public enum ActionType { + PULL((byte) 0, new PullActionReaderWriter()), + RANGE_ACK((byte) 1, new RangeAckActionReaderWriter()), + FOREVER_OFFLINE((byte) 2, new ForeverOfflineActionReaderWriter()); + + private static final ImmutableMap INSTANCES; + + static { + final Map instances = new HashMap<>(); + for (final ActionType t : values()) { + instances.put(t.getCode(), t); + } + INSTANCES = ImmutableMap.copyOf(instances); + } + + private final byte code; + private final ActionReaderWriter readerWriter; + + ActionType(final byte code, final ActionReaderWriter readerWriter) { + this.code = code; + this.readerWriter = readerWriter; + } + + public static ActionType fromCode(final byte code) { + if (INSTANCES.containsKey(code)) { + return INSTANCES.get(code); + } + + throw new RuntimeException("unknown action type code " + code); + } + + public byte getCode() { + return code; + } + + public ActionReaderWriter getReaderWriter() { + return readerWriter; + } +} diff --git a/qmq-store/src/main/java/qunar/tc/qmq/store/AppendMessageResult.java b/qmq-store/src/main/java/qunar/tc/qmq/store/AppendMessageResult.java new file mode 100644 index 00000000..52cb3e6d --- /dev/null +++ b/qmq-store/src/main/java/qunar/tc/qmq/store/AppendMessageResult.java @@ -0,0 +1,69 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.store; + +/** + * @author keli.wang + * @since 2017/7/4 + */ +public class AppendMessageResult { + private final AppendMessageStatus status; + private final long wroteOffset; + private final int wroteBytes; + private final T additional; + + public AppendMessageResult(AppendMessageStatus status) { + this(status, 0, 0, null); + } + + public AppendMessageResult(AppendMessageStatus status, long wroteOffset, int wroteBytes) { + this(status, wroteOffset, wroteBytes, null); + } + + public AppendMessageResult(AppendMessageStatus status, long wroteOffset, int wroteBytes, T additional) { + this.status = status; + this.wroteOffset = wroteOffset; + this.wroteBytes = wroteBytes; + this.additional = additional; + } + + public AppendMessageStatus getStatus() { + return status; + } + + public long getWroteOffset() { + return wroteOffset; + } + + public int getWroteBytes() { + return wroteBytes; + } + + public T getAdditional() { + return additional; + } + + @Override + public String toString() { + return "AppendMessageResult{" + + "status=" + status + + ", wroteOffset=" + wroteOffset + + ", wroteBytes=" + wroteBytes + + ", additional=" + additional + + '}'; + } +} diff --git a/qmq-store/src/main/java/qunar/tc/qmq/store/AppendMessageStatus.java b/qmq-store/src/main/java/qunar/tc/qmq/store/AppendMessageStatus.java new file mode 100644 index 00000000..b2b2e81d --- /dev/null +++ b/qmq-store/src/main/java/qunar/tc/qmq/store/AppendMessageStatus.java @@ -0,0 +1,30 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.store; + +/** + * @author keli.wang + * @since 2017/7/4 + */ +public enum AppendMessageStatus { + SUCCESS, + END_OF_FILE, + DATA_OVERFLOW, + APPEND_FAILED, + MESSAGE_SIZE_EXCEEDED, + UNKNOWN_ERROR +} diff --git a/qmq-store/src/main/java/qunar/tc/qmq/store/CheckpointLoader.java b/qmq-store/src/main/java/qunar/tc/qmq/store/CheckpointLoader.java new file mode 100644 index 00000000..6834e452 --- /dev/null +++ b/qmq-store/src/main/java/qunar/tc/qmq/store/CheckpointLoader.java @@ -0,0 +1,27 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.store; + +import io.netty.buffer.ByteBuf; + +/** + * @author keli.wang + * @since 2018/10/29 + */ +public interface CheckpointLoader { + ByteBuf loadCheckpoint(); +} diff --git a/qmq-store/src/main/java/qunar/tc/qmq/store/CheckpointManager.java b/qmq-store/src/main/java/qunar/tc/qmq/store/CheckpointManager.java new file mode 100644 index 00000000..87092aad --- /dev/null +++ b/qmq-store/src/main/java/qunar/tc/qmq/store/CheckpointManager.java @@ -0,0 +1,460 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.store; + +import com.google.common.collect.HashBasedTable; +import com.google.common.collect.Table; +import io.netty.buffer.ByteBuf; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import qunar.tc.qmq.meta.BrokerRole; +import qunar.tc.qmq.monitor.QMon; +import qunar.tc.qmq.store.action.PullAction; +import qunar.tc.qmq.store.action.RangeAckAction; + +import java.util.Collection; +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.locks.Lock; +import java.util.concurrent.locks.ReentrantLock; + +/** + * @author keli.wang + * @since 2017/8/21 + */ +public class CheckpointManager implements AutoCloseable { + private static final Logger LOG = LoggerFactory.getLogger(CheckpointManager.class); + + private final MessageCheckpointSerde messageCheckpointSerde; + private final ActionCheckpointSerde actionCheckpointSerde; + private final SnapshotStore messageCheckpointStore; + private final SnapshotStore actionCheckpointStore; + + private final Lock messageCheckpointGuard; + private final Lock actionCheckpointGuard; + private final MessageCheckpoint messageCheckpoint; + private final ActionCheckpoint actionCheckpoint; + + CheckpointManager(final BrokerRole role, final StorageConfig config, final CheckpointLoader loader) { + this.messageCheckpointSerde = new MessageCheckpointSerde(); + this.actionCheckpointSerde = new ActionCheckpointSerde(); + + this.messageCheckpointStore = new SnapshotStore<>("message-checkpoint", config, messageCheckpointSerde); + this.actionCheckpointStore = new SnapshotStore<>("action-checkpoint", config, actionCheckpointSerde); + + this.messageCheckpointGuard = new ReentrantLock(); + this.actionCheckpointGuard = new ReentrantLock(); + + final MessageCheckpoint messageCheckpoint = loadMessageCheckpoint(); + final ActionCheckpoint actionCheckpoint = loadActionCheckpoint(); + if (needSyncCheckpoint(role, messageCheckpoint, actionCheckpoint)) { + // TODO(keli.wang): must try to cleanup this messy... + final ByteBuf buf = loader.loadCheckpoint(); + buf.readByte(); + final int messageLength = buf.readInt(); + final byte[] message = new byte[messageLength]; + buf.readBytes(message); + this.messageCheckpoint = messageCheckpointSerde.fromBytes(message); + final int actionLength = buf.readInt(); + final byte[] action = new byte[actionLength]; + buf.readBytes(action); + this.actionCheckpoint = actionCheckpointSerde.fromBytes(action); + } else { + this.messageCheckpoint = messageCheckpoint; + this.actionCheckpoint = actionCheckpoint; + } + } + + private MessageCheckpoint loadMessageCheckpoint() { + final Snapshot snapshot = messageCheckpointStore.latestSnapshot(); + if (snapshot == null) { + LOG.info("no message log replay snapshot, return empty state."); + return new MessageCheckpoint(-1, new HashMap<>()); + } else { + final MessageCheckpoint checkpoint = snapshot.getData(); + if (checkpoint.isFromOldVersion()) { + checkpoint.setOffset(snapshot.getVersion()); + } + return checkpoint; + } + } + + private ActionCheckpoint loadActionCheckpoint() { + final Snapshot snapshot = actionCheckpointStore.latestSnapshot(); + if (snapshot == null) { + LOG.info("no action log replay snapshot, return empty state."); + return new ActionCheckpoint(-1, HashBasedTable.create()); + } else { + final ActionCheckpoint checkpoint = snapshot.getData(); + if (checkpoint.isFromOldVersion()) { + checkpoint.setOffset(snapshot.getVersion()); + } + return checkpoint; + } + } + + private boolean needSyncCheckpoint(final BrokerRole role, final MessageCheckpoint messageCheckpoint, final ActionCheckpoint actionCheckpoint) { + if (role != BrokerRole.SLAVE) { + return false; + } + + return messageCheckpoint.getOffset() < 0 && actionCheckpoint.getOffset() < 0; + } + + void fixOldVersionCheckpointIfShould(ConsumerLogManager consumerLogManager, PullLogManager pullLogManager) { + if (messageCheckpoint.isFromOldVersion()) { + LOG.info("fix message replay state using consumer log"); + fixMessageCheckpoint(consumerLogManager); + messageCheckpoint.setFromOldVersion(false); + } + + if (actionCheckpoint.isFromOldVersion()) { + LOG.info("fix action replay state using pull log"); + fixActionCheckpoint(pullLogManager); + messageCheckpoint.setFromOldVersion(false); + } + } + + private void fixMessageCheckpoint(ConsumerLogManager manager) { + final Map maxSequences = messageCheckpoint.getMaxSequences(); + final Map offsets = manager.currentConsumerLogOffset(); + offsets.forEach(maxSequences::put); + } + + private void fixActionCheckpoint(PullLogManager manager) { + final Table allLogs = manager.getLogs(); + final Table progresses = actionCheckpoint.getProgresses(); + progresses.values().forEach(progress -> { + final String subject = progress.getSubject(); + final String group = progress.getGroup(); + final String groupAndSubject = GroupAndSubject.groupAndSubject(subject, group); + + final Map consumers = progress.getConsumers(); + + final Map logs = allLogs.column(groupAndSubject); + logs.forEach((consumerId, log) -> { + final long pull = log.getMaxOffset() - 1; + final ConsumerProgress consumer = consumers.get(consumerId); + if (consumer != null) { + consumer.setPull(pull); + } else { + consumers.put(consumerId, new ConsumerProgress(subject, group, consumerId, pull, -1)); + } + }); + + if (consumers.size() == 1) { + consumers.values().forEach(consumer -> { + if (consumer.getPull() < 0) { + progress.setBroadcast(true); + consumer.setPull(progress.getPull()); + } + }); + } + }); + } + + Collection allConsumerGroupProgresses() { + actionCheckpointGuard.lock(); + try { + return actionCheckpoint.getProgresses().values(); + } finally { + actionCheckpointGuard.unlock(); + } + } + + ConsumerGroupProgress getConsumerGroupProgress(String subject, String group) { + actionCheckpointGuard.lock(); + try { + return actionCheckpoint.getProgresses().get(subject, group); + } finally { + actionCheckpointGuard.unlock(); + } + } + + + long getMaxPulledMessageSequence(final String subject, final String group) { + actionCheckpointGuard.lock(); + try { + final ConsumerGroupProgress progress = actionCheckpoint.getProgresses().get(subject, group); + if (progress == null) { + return -1; + } + + return progress.getPull(); + } finally { + actionCheckpointGuard.unlock(); + } + } + + public void updateActionReplayState(final long offset, final PullAction action) { + actionCheckpointGuard.lock(); + try { + updateMaxPulledMessageSequence(action); + updateConsumerMaxPullLogSequence(action); + actionCheckpoint.setOffset(offset); + } finally { + actionCheckpointGuard.unlock(); + } + } + + private void updateMaxPulledMessageSequence(final PullAction action) { + final String subject = action.subject(); + final String group = action.group(); + + final long maxSequence = getMaxPulledMessageSequence(subject, group); + if (maxSequence + 1 < action.getFirstMessageSequence()) { + long num = action.getFirstMessageSequence() - maxSequence; + LOG.warn("Maybe lost message. Last message sequence: {}. Current start sequence {} {}:{}", maxSequence, action.getFirstMessageSequence(), subject, group); + QMon.maybeLostMessagesCountInc(subject, group, num); + } + final long lastMessageSequence = action.getLastMessageSequence(); + if (maxSequence < lastMessageSequence) { + updateMaxPulledMessageSequence(subject, group, action.isBroadcast(), lastMessageSequence); + } + } + + private void updateMaxPulledMessageSequence(final String subject, final String group, final boolean broadcast, final long maxSequence) { + final ConsumerGroupProgress progress = getOrCreateConsumerGroupProgress(subject, group, broadcast); + progress.setPull(maxSequence); + } + + private void updateConsumerMaxPullLogSequence(final PullAction action) { + final String subject = action.subject(); + final String group = action.group(); + final String consumerId = action.consumerId(); + + final long maxSequence = getConsumerMaxPullLogSequence(subject, group, consumerId); + if (maxSequence + 1 < action.getFirstSequence()) { + LOG.warn("Pull log not continuous. Last pull log sequence: {}. Current start pull log sequence {} {}:{}:{}", maxSequence, action.getFirstSequence(), subject, group, consumerId); + } + + final long lastSequence = action.getLastSequence(); + if (maxSequence < lastSequence) { + updateConsumerMaxPullLogSequence(subject, group, consumerId, action.isBroadcast(), lastSequence); + } + } + + private long getConsumerMaxPullLogSequence(final String subject, final String group, final String consumerId) { + final ConsumerProgress consumer = getConsumerProgress(subject, group, consumerId); + if (consumer == null) { + return -1; + } else { + return consumer.getPull(); + } + } + + private void updateConsumerMaxPullLogSequence(final String subject, final String group, final String consumerId, final boolean broadcast, final long maxSequence) { + final ConsumerProgress consumer = getOrCreateConsumerProgress(subject, group, consumerId, broadcast); + consumer.setPull(maxSequence); + } + + private ConsumerProgress getOrCreateConsumerProgress(final String subject, final String group, final String consumerId, final boolean broadcast) { + final ConsumerGroupProgress progress = getOrCreateConsumerGroupProgress(subject, group, broadcast); + + final Map consumers = progress.getConsumers(); + if (!consumers.containsKey(consumerId)) { + consumers.put(consumerId, new ConsumerProgress(subject, group, consumerId, -1, -1)); + } + return consumers.get(consumerId); + } + + public void updateActionReplayState(final long offset, final RangeAckAction action) { + actionCheckpointGuard.lock(); + try { + final String subject = action.subject(); + final String group = action.group(); + final String consumerId = action.consumerId(); + + final long maxSequence = getConsumerMaxAckedPullLogSequence(subject, group, consumerId); + if (maxSequence + 1 < action.getFirstSequence()) { + LOG.warn("Maybe lost ack. Last acked sequence: {}. Current start acked sequence {} {}:{}:{}", maxSequence, action.getFirstSequence(), subject, group, consumerId); + } + + final long lastSequence = action.getLastSequence(); + if (maxSequence < lastSequence) { + updateConsumerMaxAckedPullLogSequence(subject, group, consumerId, lastSequence); + } + + actionCheckpoint.setOffset(offset); + } finally { + actionCheckpointGuard.unlock(); + } + } + + private long getConsumerMaxAckedPullLogSequence(final String subject, final String group, final String consumerId) { + final ConsumerProgress consumer = getConsumerProgress(subject, group, consumerId); + if (consumer == null) { + return -1; + } else { + return consumer.getAck(); + } + } + + private void updateConsumerMaxAckedPullLogSequence(final String subject, final String group, final String consumerId, final long maxSequence) { + final ConsumerProgress consumer = getConsumerProgress(subject, group, consumerId); + if (consumer != null) { + consumer.setAck(maxSequence); + } + } + + private ConsumerProgress getConsumerProgress(final String subject, final String group, final String consumerId) { + final ConsumerGroupProgress progress = actionCheckpoint.getProgresses().get(subject, group); + if (progress == null) { + return null; + } + + final Map consumers = progress.getConsumers(); + if (consumers == null) { + return null; + } + + return consumers.get(consumerId); + } + + private ConsumerGroupProgress getOrCreateConsumerGroupProgress(final String subject, final String group, final boolean broadcast) { + final Table progresses = actionCheckpoint.getProgresses(); + if (!progresses.contains(subject, group)) { + final ConsumerGroupProgress progress = new ConsumerGroupProgress(subject, group, broadcast, -1, new HashMap<>()); + progresses.put(subject, group, progress); + + } + return progresses.get(subject, group); + } + + void removeConsumerProgress(String subject, String group, String consumerId) { + final ConsumerGroupProgress progress = actionCheckpoint.getProgresses().get(subject, group); + if (progress == null) { + return; + } + + final Map consumers = progress.getConsumers(); + if (consumers != null) { + consumers.remove(consumerId); + } + } + + void updateMessageReplayState(final MessageLogMeta meta) { + messageCheckpointGuard.lock(); + try { + final String subject = meta.getSubject(); + final long sequence = meta.getSequence(); + + final Map sequences = messageCheckpoint.getMaxSequences(); + if (sequences.containsKey(subject)) { + sequences.merge(subject, sequence, Math::max); + } else { + sequences.put(subject, sequence); + } + + final long offset = meta.getWroteOffset() + meta.getWroteBytes(); + messageCheckpoint.setOffset(offset); + } finally { + messageCheckpointGuard.unlock(); + } + } + + public byte[] dumpMessageCheckpoint() { + messageCheckpointGuard.lock(); + try { + return messageCheckpointSerde.toBytes(messageCheckpoint); + } finally { + messageCheckpointGuard.unlock(); + } + } + + public byte[] dumpActionCheckpoint() { + actionCheckpointGuard.lock(); + try { + return actionCheckpointSerde.toBytes(actionCheckpoint); + } finally { + actionCheckpointGuard.unlock(); + } + } + + Snapshot createMessageCheckpointSnapshot() { + final MessageCheckpoint checkpoint = duplicateMessageCheckpoint(); + return new Snapshot<>(checkpoint.getOffset(), checkpoint); + } + + Snapshot createActionCheckpointSnapshot() { + final ActionCheckpoint checkpoint = duplicateActionCheckpoint(); + return new Snapshot<>(checkpoint.getOffset(), checkpoint); + } + + private MessageCheckpoint duplicateMessageCheckpoint() { + messageCheckpointGuard.lock(); + try { + return new MessageCheckpoint(messageCheckpoint.getOffset(), new HashMap<>(messageCheckpoint.getMaxSequences())); + } finally { + messageCheckpointGuard.unlock(); + } + } + + private ActionCheckpoint duplicateActionCheckpoint() { + actionCheckpointGuard.lock(); + try { + final Table progresses = HashBasedTable.create(); + for (final ConsumerGroupProgress progress : actionCheckpoint.getProgresses().values()) { + final Map consumers = progress.getConsumers(); + if (consumers == null) { + continue; + } + + final Map consumersCopy = new HashMap<>(); + for (final ConsumerProgress consumer : consumers.values()) { + consumersCopy.put(consumer.getConsumerId(), new ConsumerProgress(consumer)); + } + final String subject = progress.getSubject(); + final String group = progress.getGroup(); + progresses.put(subject, group, new ConsumerGroupProgress(subject, group, progress.isBroadcast(), progress.getPull(), consumersCopy)); + } + final long offset = actionCheckpoint.getOffset(); + return new ActionCheckpoint(offset, progresses); + } finally { + actionCheckpointGuard.unlock(); + } + } + + // TODO(keli.wang): update offset and state at the same time within the lock + public long getActionCheckpointOffset() { + return actionCheckpoint.getOffset(); + } + + public long getMessageCheckpointOffset() { + return messageCheckpoint.getOffset(); + } + + void saveMessageCheckpointSnapshot(final Snapshot snapshot) { + if (snapshot.getVersion() < 0) { + return; + } + messageCheckpointStore.saveSnapshot(snapshot); + } + + void saveActionCheckpointSnapshot(final Snapshot snapshot) { + if (snapshot.getVersion() < 0) { + return; + } + actionCheckpointStore.saveSnapshot(snapshot); + } + + @Override + public void close() { + messageCheckpointStore.close(); + actionCheckpointStore.close(); + } +} diff --git a/qmq-store/src/main/java/qunar/tc/qmq/store/CheckpointStore.java b/qmq-store/src/main/java/qunar/tc/qmq/store/CheckpointStore.java new file mode 100644 index 00000000..1d5a3bd3 --- /dev/null +++ b/qmq-store/src/main/java/qunar/tc/qmq/store/CheckpointStore.java @@ -0,0 +1,139 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.store; + +import com.google.common.base.Preconditions; +import com.google.common.io.Files; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.io.IOException; + +/** + * @author keli.wang + * @since 2017/7/7 + */ +public class CheckpointStore { + private static final Logger LOG = LoggerFactory.getLogger(CheckpointStore.class); + + private final String storePath; + private final String filename; + private final Serde serde; + + public CheckpointStore(final String storePath, final String filename, final Serde serde) { + this.storePath = storePath; + this.filename = filename; + this.serde = serde; + + ensureDir(storePath); + } + + private void ensureDir(final String storePath) { + final File store = new File(storePath); + if (store.exists()) { + return; + } + + final boolean success = store.mkdirs(); + if (!success) { + throw new RuntimeException("Failed create path " + storePath); + } + LOG.info("Create checkpoint store {} success.", storePath); + } + + public T loadCheckpoint() { + final File checkpointFile = checkpointFile(); + final File backupFile = backupFile(); + + if (!checkpointFile.exists() && !backupFile.exists()) { + LOG.warn("Checkpoint file and backup file does not exist, return null for now"); + return null; + } + + try { + final byte[] data = Files.toByteArray(checkpointFile); + if (data != null && data.length == 0) { + return null; + } + return serde.fromBytes(data); + } catch (IOException e) { + LOG.error("Load checkpoint file failed. Try load backup checkpoint file instead.", e); + } + + try { + return serde.fromBytes(Files.toByteArray(backupFile)); + } catch (IOException e) { + LOG.error("Load backup checkpoint file failed.", e); + } + + throw new RuntimeException("Load checkpoint failed. filename=" + filename); + } + + // TODO(keli.wang): 根据数据量大小看看后面是不是需要进行压缩,毕竟数据量大了之后一直保存可能很耗IO + public void saveCheckpoint(final T checkpoint) { + final byte[] data = serde.toBytes(checkpoint); + Preconditions.checkState(data != null, "Serialized checkpoint data should not be null."); + if (data.length == 0) { + return; + } + + final File tmp = tmpFile(); + try { + Files.write(data, tmp); + } catch (IOException e) { + LOG.error("write data into tmp checkpoint file failed. file={}", tmp, e); + throw new RuntimeException("write checkpoint data failed.", e); + } + + final File checkpointFile = checkpointFile(); + if (checkpointFile.exists()) { + final File backupFile = backupFile(); + + if (backupFile.exists() && !backupFile.delete()) { + LOG.error("Delete backup file failed.backup: {}", backupFile); + } + + if (!checkpointFile.renameTo(backupFile)) { + LOG.error("Backup current checkpoint file failed. checkpoint:{}, backup: {}", checkpointFile, backupFile); + throw new RuntimeException("Backup current checkpoint file failed"); + } + + if (checkpointFile.exists() && !checkpointFile.delete()) { + LOG.error("Delete checkpoint file failed.backup: {}", backupFile); + } + } + + if (!tmp.renameTo(checkpointFile)) { + LOG.error("Move tmp as checkpoint file failed. tmp:{}, checkpoint: {}", tmp, checkpointFile); + throw new RuntimeException("Move tmp as checkpoint file failed."); + } + } + + private File checkpointFile() { + return new File(storePath, filename); + } + + private File tmpFile() { + return new File(storePath, filename + ".tmp"); + } + + private File backupFile() { + return new File(storePath, filename + ".backup"); + } + +} diff --git a/qmq-store/src/main/java/qunar/tc/qmq/store/ConsumeQueue.java b/qmq-store/src/main/java/qunar/tc/qmq/store/ConsumeQueue.java new file mode 100644 index 00000000..0c539e22 --- /dev/null +++ b/qmq-store/src/main/java/qunar/tc/qmq/store/ConsumeQueue.java @@ -0,0 +1,99 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.store; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import qunar.tc.qmq.monitor.QMon; +import qunar.tc.qmq.utils.RetrySubjectUtils; + +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicLong; + +/** + * 维护每个消费组的消费进度 + * nextSequence即下一次拉取的时候对应的consumer log上的sequence + * + * @author keli.wang + * @since 2017/7/31 + */ +public class ConsumeQueue { + private static final Logger LOG = LoggerFactory.getLogger(ConsumeQueue.class); + + private final Storage storage; + private final String subject; + private final String group; + private final AtomicLong nextSequence; + private final AtomicBoolean monitorEnabled = new AtomicBoolean(false); + + public ConsumeQueue(final Storage storage, final String subject, final String group, final long lastMaxSequence) { + this.storage = storage; + this.subject = subject; + this.group = group; + this.nextSequence = new AtomicLong(lastMaxSequence + 1); + } + + public synchronized void setNextSequence(long nextSequence) { + this.nextSequence.set(nextSequence); + } + + public long getQueueCount() { + return storage.getMaxMessageSequence(subject) - nextSequence.get(); + } + + public synchronized GetMessageResult pollMessages(final int maxMessages) { + enableLagMonitor(); + + long currentSequence = nextSequence.get(); + if (RetrySubjectUtils.isRetrySubject(subject)) { + return storage.pollMessages(subject, currentSequence, maxMessages, this::isDelayReached); + } else { + final GetMessageResult result = storage.pollMessages(subject, currentSequence, maxMessages); + long actualSequence = result.getNextBeginOffset() - result.getSegmentBuffers().size(); + long delta = actualSequence - currentSequence; + if (delta > 0) { + QMon.expiredMessagesCountInc(subject, group, delta); + LOG.error("next sequence skipped. subject: {}, group: {}, nextSequence: {}, result: {}", subject, group, currentSequence, result); + } + return result; + } + } + + private boolean isDelayReached(ConsumerLogEntry entry) { + final int delayMillis = storage.getStorageConfig().getRetryDelaySeconds() * 1000; + return entry.getTimestamp() + delayMillis <= System.currentTimeMillis(); + } + + private void enableLagMonitor() { + try { + if (monitorEnabled.compareAndSet(false, true)) { + QMon.messageSequenceLagGauge(subject, group, () -> (double) getQueueCount()); + LOG.info("enable message sequence lag monitor:{} {}", subject, group); + } + } catch (Throwable e) { + LOG.error("enable message sequence lag monitor error:{} {}", subject, group, e); + } + } + + void disableLagMonitor(String subject, String group) { + if (monitorEnabled.compareAndSet(true, false)) { + // TODO(keli.wang): can we avoid remove this metrics by clean up all useless data after offline all consumers in this group? + QMon.removeMessageSequenceLag(subject, group); + LOG.info("disable message sequence lag monitor:{} {}", subject, group); + } + } +} diff --git a/qmq-store/src/main/java/qunar/tc/qmq/store/ConsumeQueueManager.java b/qmq-store/src/main/java/qunar/tc/qmq/store/ConsumeQueueManager.java new file mode 100644 index 00000000..fca7d6b9 --- /dev/null +++ b/qmq-store/src/main/java/qunar/tc/qmq/store/ConsumeQueueManager.java @@ -0,0 +1,74 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.store; + +import com.google.common.collect.HashBasedTable; +import com.google.common.collect.Table; + +import java.util.Collections; +import java.util.Map; + +/** + * @author keli.wang + * @since 2017/7/31 + */ +public class ConsumeQueueManager { + private final Table queues; + private final Storage storage; + + public ConsumeQueueManager(final Storage storage) { + this.queues = HashBasedTable.create(); + this.storage = storage; + } + + public synchronized ConsumeQueue getOrCreate(final String subject, final String group) { + if (!queues.contains(subject, group)) { + queues.put(subject, group, new ConsumeQueue(storage, subject, group, getLastMaxSequence(subject, group))); + } + return queues.get(subject, group); + } + + public synchronized Map getBySubject(final String subject) { + if (queues.containsRow(subject)) { + return queues.row(subject); + } else { + return Collections.emptyMap(); + } + } + + private long getLastMaxSequence(final String subject, final String group) { + final ConsumerGroupProgress progress = storage.getConsumerGroupProgress(subject, group); + if (progress == null) { + return -1; + } else { + return progress.getPull(); + } + } + + public synchronized void update(final String subject, final String group, final long nextSequence) { + final ConsumeQueue queue = getOrCreate(subject, group); + queue.setNextSequence(nextSequence); + } + + synchronized void disableLagMonitor(String subject, String group) { + final ConsumeQueue consumeQueue = queues.get(subject, group); + if (consumeQueue == null) { + return; + } + consumeQueue.disableLagMonitor(subject, group); + } +} diff --git a/qmq-store/src/main/java/qunar/tc/qmq/store/ConsumerGroupProgress.java b/qmq-store/src/main/java/qunar/tc/qmq/store/ConsumerGroupProgress.java new file mode 100644 index 00000000..0feef2bb --- /dev/null +++ b/qmq-store/src/main/java/qunar/tc/qmq/store/ConsumerGroupProgress.java @@ -0,0 +1,68 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.store; + +import java.util.Map; + +/** + * @author keli.wang + * @since 2018/10/24 + */ +public class ConsumerGroupProgress { + private final String subject; + private final String group; + private final Map consumers; + // TODO(keli.wang): mark broadcast as final after new snapshot file created + private boolean broadcast; + private long pull; + + public ConsumerGroupProgress(String subject, String group, boolean broadcast, long pull, Map consumers) { + this.subject = subject; + this.group = group; + this.broadcast = broadcast; + this.pull = pull; + this.consumers = consumers; + } + + public String getSubject() { + return subject; + } + + public String getGroup() { + return group; + } + + public boolean isBroadcast() { + return broadcast; + } + + public void setBroadcast(boolean broadcast) { + this.broadcast = broadcast; + } + + public Map getConsumers() { + return consumers; + } + + public long getPull() { + return pull; + } + + public void setPull(long pull) { + this.pull = pull; + } +} diff --git a/qmq-store/src/main/java/qunar/tc/qmq/store/ConsumerLog.java b/qmq-store/src/main/java/qunar/tc/qmq/store/ConsumerLog.java new file mode 100644 index 00000000..e6a646de --- /dev/null +++ b/qmq-store/src/main/java/qunar/tc/qmq/store/ConsumerLog.java @@ -0,0 +1,236 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.store; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import qunar.tc.qmq.monitor.QMon; + +import java.io.File; +import java.nio.ByteBuffer; +import java.util.concurrent.locks.ReentrantLock; + +/** + * @author keli.wang + * @since 2017/7/5 + */ +public class ConsumerLog { + private static final Logger LOG = LoggerFactory.getLogger(ConsumerLog.class); + + // 8 bytes timestamp + 8 bytes wrote offset + 4 bytes wrote bytes + 2 bytes header size + static final int CONSUMER_LOG_UNIT_BYTES = 22; + private static final int CONSUMER_LOG_SIZE = CONSUMER_LOG_UNIT_BYTES * 10000000; + + private final StorageConfig config; + private final String subject; + private final LogManager logManager; + private final MessageAppender consumerLogAppender = new ConsumerLogMessageAppender(); + private final ReentrantLock putMessageLogOffsetLock = new ReentrantLock(); + + private volatile long minSequence; + + public ConsumerLog(final StorageConfig config, final String subject) { + this.config = config; + this.subject = subject; + this.logManager = new LogManager(new File(config.getConsumerLogStorePath(), subject), CONSUMER_LOG_SIZE, config, new ConsumerLogSegmentValidator()); + } + + // TODO(keli.wang): handle write fail and retry + public boolean putMessageLogOffset(final long sequence, final long offset, final int size, final short headerSize) { + putMessageLogOffsetLock.lock(); + try { + if (sequence < nextSequence()) { + return true; + } + + final long expectedOffset = sequence * CONSUMER_LOG_UNIT_BYTES; + LogSegment segment = logManager.locateSegment(expectedOffset); + if (segment == null) { + segment = logManager.allocOrResetSegments(expectedOffset); + } + fillPreBlank(segment, expectedOffset); + + final AppendMessageResult result = segment.append(new ConsumerLogMessage(sequence, offset, size, headerSize), consumerLogAppender); + switch (result.getStatus()) { + case SUCCESS: + break; + case END_OF_FILE: + logManager.allocNextSegment(); + return putMessageLogOffset(sequence, offset, size, headerSize); + default: + return false; + } + } finally { + putMessageLogOffsetLock.unlock(); + } + + return true; + } + + private void fillPreBlank(final LogSegment segment, final long untilWhere) { + final ConsumerLogMessage blankMessage = new ConsumerLogMessage(0, 0, Integer.MAX_VALUE, (short) 0); + final long startOffset = segment.getBaseOffset() + segment.getWrotePosition(); + for (long i = startOffset; i < untilWhere; i += CONSUMER_LOG_UNIT_BYTES) { + segment.append(blankMessage, consumerLogAppender); + } + } + + public SegmentBuffer selectIndexBuffer(long startIndex) { + final long startOffset = startIndex * CONSUMER_LOG_UNIT_BYTES; + final LogSegment segment = logManager.locateSegment(startOffset); + if (segment == null) { + QMon.hitDeletedConsumerLogSegmentCountInc(subject); + return null; + } else { + return segment.selectSegmentBuffer((int) (startOffset % CONSUMER_LOG_SIZE)); + } + } + + public void setMinSequence(long sequence) { + long computedMinSequence = getMinOffset(); + if (computedMinSequence < sequence) { + this.minSequence = sequence; + QMon.adjustConsumerLogMinOffset(subject); + LOG.info("adjust consumer log {} min offset from {} to {}.", subject, computedMinSequence, minSequence); + } + } + + public long getMinOffset() { + long computedMinSequence = logManager.getMinOffset() / CONSUMER_LOG_UNIT_BYTES; + if (computedMinSequence < minSequence) { + return minSequence; + } else { + return computedMinSequence; + } + } + + public long nextSequence() { + return logManager.getMaxOffset() / CONSUMER_LOG_UNIT_BYTES; + } + + public OffsetBound getOffsetBound() { + final long minOffset = getMinOffset(); + final long maxOffset = nextSequence(); + return new OffsetBound(Math.min(minOffset, maxOffset), maxOffset); + } + + public void flush() { + logManager.flush(); + QMon.flushConsumerLogCountInc(subject); + } + + public void close() { + logManager.close(); + } + + public void clean() { + logManager.deleteExpiredSegments(config.getConsumerLogRetentionMs()); + } + + private static class ConsumerLogMessage { + private final long sequence; + private final long offset; + private final int size; + private final short headerSize; + + private ConsumerLogMessage(long sequence, long offset, int size, short headerSize) { + this.sequence = sequence; + this.offset = offset; + this.size = size; + this.headerSize = headerSize; + } + + public long getSequence() { + return sequence; + } + + public long getOffset() { + return offset; + } + + public int getSize() { + return size; + } + + public short getHeaderSize() { + return headerSize; + } + } + + private static class ConsumerLogMessageAppender implements MessageAppender { + private final ByteBuffer workingBuffer = ByteBuffer.allocate(CONSUMER_LOG_UNIT_BYTES); + + @Override + public AppendMessageResult doAppend(long baseOffset, ByteBuffer targetBuffer, int freeSpace, ConsumerLogMessage message) { + workingBuffer.clear(); + + final long wroteOffset = baseOffset + targetBuffer.position(); + workingBuffer.flip(); + workingBuffer.limit(CONSUMER_LOG_UNIT_BYTES); + workingBuffer.putLong(System.currentTimeMillis()); + workingBuffer.putLong(message.getOffset()); + workingBuffer.putInt(message.getSize()); + workingBuffer.putShort(message.getHeaderSize()); + targetBuffer.put(workingBuffer.array(), 0, CONSUMER_LOG_UNIT_BYTES); + return new AppendMessageResult<>(AppendMessageStatus.SUCCESS, wroteOffset, CONSUMER_LOG_UNIT_BYTES); + } + } + + private static class ConsumerLogSegmentValidator implements LogSegmentValidator { + @Override + public ValidateResult validate(LogSegment segment) { + final int fileSize = segment.getFileSize(); + final ByteBuffer buffer = segment.sliceByteBuffer(); + + int position = 0; + while (true) { + if (position == fileSize) { + return new ValidateResult(ValidateStatus.COMPLETE, fileSize); + } + + final int result = consumeAndValidateMessage(buffer); + if (result == -1) { + return new ValidateResult(ValidateStatus.PARTIAL, position); + } else { + position += result; + } + } + } + + private int consumeAndValidateMessage(final ByteBuffer buffer) { + final long timestamp = buffer.getLong(); + final long offset = buffer.getLong(); + final int size = buffer.getInt(); + final short headerSize = buffer.getShort(); + if (isBlankMessage(timestamp, offset, size, headerSize)) { + return CONSUMER_LOG_UNIT_BYTES; + } else if (isValidMessage(timestamp, offset, size, headerSize)) { + return CONSUMER_LOG_UNIT_BYTES; + } else { + return -1; + } + } + + private boolean isBlankMessage(final long timestamp, final long offset, final int size, final short headerSize) { + return timestamp > 0 && offset == 0 && headerSize == 0 && size == Integer.MAX_VALUE; + } + + private boolean isValidMessage(final long timestamp, final long offset, final int size, final short headerSize) { + return timestamp > 0 && offset >= 0 && size > 0 && headerSize <= size && headerSize >= MessageLog.MIN_RECORD_BYTES; + } + } +} diff --git a/qmq-store/src/main/java/qunar/tc/qmq/store/ConsumerLogEntry.java b/qmq-store/src/main/java/qunar/tc/qmq/store/ConsumerLogEntry.java new file mode 100644 index 00000000..2393b4e1 --- /dev/null +++ b/qmq-store/src/main/java/qunar/tc/qmq/store/ConsumerLogEntry.java @@ -0,0 +1,68 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.store; + +/** + * Created by zhaohui.yu + * 9/3/18 + */ +public class ConsumerLogEntry { + private long timestamp; + private long wroteOffset; + private int wroteBytes; + private short headerSize; + + public long getTimestamp() { + return timestamp; + } + + public void setTimestamp(long timestamp) { + this.timestamp = timestamp; + } + + public long getWroteOffset() { + return wroteOffset; + } + + public void setWroteOffset(long wroteOffset) { + this.wroteOffset = wroteOffset; + } + + public int getWroteBytes() { + return wroteBytes; + } + + public void setWroteBytes(int wroteBytes) { + this.wroteBytes = wroteBytes; + } + + public short getHeaderSize() { + return headerSize; + } + + public void setHeaderSize(short headerSize) { + this.headerSize = headerSize; + } + + public static class Factory { + private static final ThreadLocal ENTRY = ThreadLocal.withInitial(() -> new ConsumerLogEntry()); + + public static ConsumerLogEntry create() { + return ENTRY.get(); + } + } +} diff --git a/qmq-store/src/main/java/qunar/tc/qmq/store/ConsumerLogFlusher.java b/qmq-store/src/main/java/qunar/tc/qmq/store/ConsumerLogFlusher.java new file mode 100644 index 00000000..2b868a35 --- /dev/null +++ b/qmq-store/src/main/java/qunar/tc/qmq/store/ConsumerLogFlusher.java @@ -0,0 +1,109 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.store; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import qunar.tc.qmq.concurrent.NamedThreadFactory; +import qunar.tc.qmq.monitor.QMon; +import qunar.tc.qmq.store.event.FixedExecOrderEventBus; + +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicLong; + +/** + * @author keli.wang + * @since 2018/9/12 + */ +public class ConsumerLogFlusher implements FixedExecOrderEventBus.Listener, AutoCloseable { + private static final Logger LOG = LoggerFactory.getLogger(ConsumerLogFlusher.class); + + private final StorageConfig config; + private final CheckpointManager checkpointManager; + private final ConsumerLogManager consumerLogManager; + private final ScheduledExecutorService flushExecutor; + private final AtomicLong counter; + private volatile long latestFlushTime; + + public ConsumerLogFlusher(final StorageConfig config, final CheckpointManager checkpointManager, final ConsumerLogManager consumerLogManager) { + this.config = config; + this.checkpointManager = checkpointManager; + this.consumerLogManager = consumerLogManager; + this.flushExecutor = Executors.newSingleThreadScheduledExecutor(new NamedThreadFactory("consumer-log-flusher")); + this.counter = new AtomicLong(0); + this.latestFlushTime = -1; + + scheduleForceFlushTask(); + } + + private void scheduleForceFlushTask() { + flushExecutor.scheduleWithFixedDelay(this::tryForceSubmitFlushTask, 1, 1, TimeUnit.MINUTES); + } + + private void tryForceSubmitFlushTask() { + final long interval = System.currentTimeMillis() - latestFlushTime; + if (interval < TimeUnit.MINUTES.toMillis(1)) { + return; + } + + submitFlushTask(); + } + + @Override + public void onEvent(final MessageLogMeta event) { + final long count = counter.incrementAndGet(); + if (count < config.getMessageCheckpointInterval()) { + return; + } + + QMon.consumerLogFlusherExceedCheckpointIntervalCountInc(); + submitFlushTask(); + } + + private synchronized void submitFlushTask() { + counter.set(0); + latestFlushTime = System.currentTimeMillis(); + + final Snapshot snapshot = checkpointManager.createMessageCheckpointSnapshot(); + flushExecutor.submit(() -> { + final long start = System.currentTimeMillis(); + try { + consumerLogManager.flush(); + checkpointManager.saveMessageCheckpointSnapshot(snapshot); + } catch (Exception e) { + QMon.consumerLogFlusherFlushFailedCountInc(); + LOG.error("flush consumer log failed. offset: {}", snapshot.getVersion(), e); + } finally { + QMon.consumerLogFlusherElapsedPerExecute(System.currentTimeMillis() - start); + } + }); + } + + @Override + public void close() { + LOG.info("try flush one more time before exit."); + submitFlushTask(); + flushExecutor.shutdown(); + try { + flushExecutor.awaitTermination(1, TimeUnit.MINUTES); + } catch (InterruptedException e) { + LOG.warn("interrupted during closing consumer log flusher."); + } + } +} diff --git a/qmq-store/src/main/java/qunar/tc/qmq/store/ConsumerLogManager.java b/qmq-store/src/main/java/qunar/tc/qmq/store/ConsumerLogManager.java new file mode 100644 index 00000000..8fcd4be3 --- /dev/null +++ b/qmq-store/src/main/java/qunar/tc/qmq/store/ConsumerLogManager.java @@ -0,0 +1,189 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.store; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.base.Preconditions; +import com.google.common.base.Strings; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import qunar.tc.qmq.monitor.QMon; + +import java.io.File; +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; + +/** + * @author keli.wang + * @since 2017/8/19 + */ +public class ConsumerLogManager implements AutoCloseable { + private static final Logger LOG = LoggerFactory.getLogger(ConsumerLogManager.class); + + private static final ObjectMapper MAPPER = new ObjectMapper(); + + private final StorageConfig config; + + private final ConcurrentMap logs; + private final ConcurrentMap offsets; + + ConsumerLogManager(final StorageConfig config) { + this.config = config; + this.logs = new ConcurrentHashMap<>(); + this.offsets = new ConcurrentHashMap<>(); + + loadConsumerLogs(); + } + + private void loadConsumerLogs() { + LOG.info("Start load consumer logs"); + + final File root = new File(config.getConsumerLogStorePath()); + final File[] consumerLogDirs = root.listFiles(); + if (consumerLogDirs != null) { + for (final File consumerLogDir : consumerLogDirs) { + if (!consumerLogDir.isDirectory()) { + continue; + } + + final String subject = consumerLogDir.getName(); + final ConsumerLog consumerLog = new ConsumerLog(config, subject); + logs.put(subject, consumerLog); + } + } + + LOG.info("Load consumer logs done"); + } + + void initConsumerLogOffset() { + for (Map.Entry entry : logs.entrySet()) { + offsets.put(entry.getKey(), entry.getValue().nextSequence()); + } + } + + Map currentConsumerLogOffset() { + final Map map = new HashMap<>(); + for (Map.Entry entry : logs.entrySet()) { + map.put(entry.getKey(), entry.getValue().nextSequence() - 1); + } + return map; + } + + ConsumerLog getOrCreateConsumerLog(final String subject) { + Preconditions.checkArgument(!Strings.isNullOrEmpty(subject), "message subject cannot be null or empty"); + if (!logs.containsKey(subject)) { + synchronized (logs) { + if (!logs.containsKey(subject)) { + logs.put(subject, new ConsumerLog(config, subject)); + } + } + } + + return logs.get(subject); + } + + ConsumerLog getConsumerLog(final String subject) { + Preconditions.checkArgument(!Strings.isNullOrEmpty(subject), "message subject cannot be null or empty"); + return logs.get(subject); + } + + long getOffsetOrDefault(final String subject, final long defaultVal) { + return offsets.getOrDefault(subject, defaultVal); + } + + long incOffset(final String subject) { + return offsets.compute(subject, (key, offset) -> offset == null ? 1 : offset + 1); + } + + public void flush() { + final long start = System.currentTimeMillis(); + try { + for (final ConsumerLog log : logs.values()) { + log.flush(); + } + } finally { + QMon.flushConsumerLogTimer(System.currentTimeMillis() - start); + } + } + + public void clean() { + for (final ConsumerLog log : logs.values()) { + log.clean(); + } + } + + void adjustConsumerLogMinOffset(LogSegment firstSegment) { + if (firstSegment == null) return; + + final String fileName = StoreUtils.offsetFileNameForSegment(firstSegment); + final CheckpointStore> offsetStore = new CheckpointStore<>(config.getMessageLogStorePath(), fileName, new ConsumerLogMinOffsetSerde()); + final Map offsets = offsetStore.loadCheckpoint(); + if (offsets == null) return; + + LOG.info("adjust consumer log min offset with offset file {}", fileName); + + for (Map.Entry entry : offsets.entrySet()) { + final ConsumerLog log = logs.get(entry.getKey()); + if (log == null) { + LOG.warn("cannot find consumer log {} while adjust min offset.", entry.getKey()); + } else { + long adjustedMinOffset = entry.getValue() + 1; + log.setMinSequence(adjustedMinOffset); + } + } + } + + void createOffsetFileFor(long baseOffset, Map offsets) { + final String fileName = StoreUtils.offsetFileNameOf(baseOffset); + final CheckpointStore> offsetStore = new CheckpointStore<>(config.getMessageLogStorePath(), fileName, new ConsumerLogMinOffsetSerde()); + offsetStore.saveCheckpoint(offsets); + } + + @Override + public void close() { + for (final ConsumerLog log : logs.values()) { + log.close(); + } + } + + private static class ConsumerLogMinOffsetSerde implements Serde> { + + @Override + public byte[] toBytes(Map value) { + try { + return MAPPER.writeValueAsBytes(value); + } catch (JsonProcessingException e) { + throw new RuntimeException("serialize message log min offset failed.", e); + } + } + + @Override + public Map fromBytes(byte[] data) { + try { + return MAPPER.readValue(data, new TypeReference>() { + }); + } catch (IOException e) { + throw new RuntimeException("deserialize offset checkpoint failed.", e); + } + } + } +} diff --git a/qmq-store/src/main/java/qunar/tc/qmq/store/ConsumerLogWroteEvent.java b/qmq-store/src/main/java/qunar/tc/qmq/store/ConsumerLogWroteEvent.java new file mode 100644 index 00000000..371c720f --- /dev/null +++ b/qmq-store/src/main/java/qunar/tc/qmq/store/ConsumerLogWroteEvent.java @@ -0,0 +1,39 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.store; + +/** + * @author keli.wang + * @since 2017/7/13 + */ +public class ConsumerLogWroteEvent { + private final String subject; + private final boolean success; + + public ConsumerLogWroteEvent(String subject, boolean success) { + this.subject = subject; + this.success = success; + } + + public String getSubject() { + return subject; + } + + public boolean isSuccess() { + return success; + } +} diff --git a/qmq-store/src/main/java/qunar/tc/qmq/store/ConsumerProgress.java b/qmq-store/src/main/java/qunar/tc/qmq/store/ConsumerProgress.java new file mode 100644 index 00000000..b9109500 --- /dev/null +++ b/qmq-store/src/main/java/qunar/tc/qmq/store/ConsumerProgress.java @@ -0,0 +1,109 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.store; + +/** + * 表示一个消费者的消费进度。 + * pull: 当前已经拉取了多少消息 + * ack: 当前已经Ack了多少消息 + * + * @author keli.wang + * @since 2018/10/22 + */ +public class ConsumerProgress { + private final String subject; + private final String group; + private final String consumerId; + + private long pull; + private long ack; + + public ConsumerProgress(String subject, String group, String consumerId, long pull, long ack) { + this.subject = subject; + this.group = group; + this.consumerId = consumerId; + this.pull = pull; + this.ack = ack; + } + + public ConsumerProgress(ConsumerProgress progress) { + this.subject = progress.getSubject(); + this.group = progress.getGroup(); + this.consumerId = progress.getConsumerId(); + this.pull = progress.getPull(); + this.ack = progress.getAck(); + } + + public String getSubject() { + return subject; + } + + public String getGroup() { + return group; + } + + public String getConsumerId() { + return consumerId; + } + + public long getPull() { + return pull; + } + + public void setPull(long pull) { + this.pull = pull; + } + + public long getAck() { + return ack; + } + + public void setAck(long ack) { + this.ack = ack; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + + ConsumerProgress that = (ConsumerProgress) o; + + if (!subject.equals(that.subject)) return false; + if (!group.equals(that.group)) return false; + return consumerId.equals(that.consumerId); + } + + @Override + public int hashCode() { + int result = subject.hashCode(); + result = 31 * result + group.hashCode(); + result = 31 * result + consumerId.hashCode(); + return result; + } + + @Override + public String toString() { + return "ConsumeProgress{" + + "subject='" + subject + '\'' + + ", group='" + group + '\'' + + ", consumerId='" + consumerId + '\'' + + ", pull=" + pull + + ", ack=" + ack + + '}'; + } +} diff --git a/qmq-store/src/main/java/qunar/tc/qmq/store/DataTransfer.java b/qmq-store/src/main/java/qunar/tc/qmq/store/DataTransfer.java new file mode 100644 index 00000000..57f69d9b --- /dev/null +++ b/qmq-store/src/main/java/qunar/tc/qmq/store/DataTransfer.java @@ -0,0 +1,104 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.store; + +import io.netty.channel.FileRegion; +import io.netty.util.ReferenceCounted; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.channels.GatheringByteChannel; +import java.nio.channels.WritableByteChannel; + +/** + * @author keli.wang + * @since 2017/7/6 + */ +public class DataTransfer implements FileRegion { + private final ByteBuffer headerBuffer; + private final SegmentBuffer segmentBuffer; + private final int bufferTotalSize; + + private final ByteBuffer[] buffers; + + private long transferred; + + public DataTransfer(ByteBuffer headerBuffer, SegmentBuffer segmentBuffer, int bufferTotalSize) { + + this.headerBuffer = headerBuffer; + this.segmentBuffer = segmentBuffer; + this.bufferTotalSize = bufferTotalSize; + + this.buffers = new ByteBuffer[2]; + this.buffers[0] = headerBuffer; + this.buffers[1] = segmentBuffer.getBuffer(); + } + + @Override + public long position() { + long pos = 0; + for (ByteBuffer buffer : this.buffers) { + pos += buffer.position(); + } + return pos; + } + + @Override + public long transfered() { + return transferred; + } + + @Override + public long count() { + return headerBuffer.limit() + bufferTotalSize; + } + + @Override + public long transferTo(WritableByteChannel target, long position) throws IOException { + GatheringByteChannel channel = (GatheringByteChannel) target; + long write = channel.write(this.buffers); + transferred += write; + return write; + } + + + @Override + public int refCnt() { + return 0; + } + + @Override + public ReferenceCounted retain() { + return null; + } + + @Override + public ReferenceCounted retain(int increment) { + return null; + } + + @Override + public boolean release() { + segmentBuffer.release(); + return true; + } + + @Override + public boolean release(int decrement) { + return release(); + } +} diff --git a/qmq-store/src/main/java/qunar/tc/qmq/store/DefaultStorage.java b/qmq-store/src/main/java/qunar/tc/qmq/store/DefaultStorage.java new file mode 100644 index 00000000..26ca51c0 --- /dev/null +++ b/qmq-store/src/main/java/qunar/tc/qmq/store/DefaultStorage.java @@ -0,0 +1,481 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.store; + +import com.google.common.collect.Table; +import com.google.common.util.concurrent.ThreadFactoryBuilder; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import qunar.tc.qmq.meta.BrokerRole; +import qunar.tc.qmq.base.RawMessage; +import qunar.tc.qmq.monitor.QMon; +import qunar.tc.qmq.store.action.ActionEvent; +import qunar.tc.qmq.store.action.MaxSequencesUpdater; +import qunar.tc.qmq.store.action.PullLogBuilder; +import qunar.tc.qmq.store.event.FixedExecOrderEventBus; + +import java.nio.ByteBuffer; +import java.util.Collection; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; + + +/** + * @author keli.wang + * @since 2017/7/4 + */ +public class DefaultStorage implements Storage { + private static final Logger LOG = LoggerFactory.getLogger(DefaultStorage.class); + + private static final int DEFAULT_FLUSH_INTERVAL = 500; // ms + + private final StorageConfig config; + + private final MessageLog messageLog; + private final ConsumerLogManager consumerLogManager; + private final PullLogManager pullLogManager; + private final ActionLog actionLog; + private final ConsumeQueueManager consumeQueueManager; + + private final CheckpointManager checkpointManager; + + private final PullLogFlusher pullLogFlusher; + private final FixedExecOrderEventBus actionEventBus; + private final ActionLogIterateService actionLogIterateService; + private final ConsumerLogFlusher consumerLogFlusher; + private final FixedExecOrderEventBus messageEventBus; + private final MessageLogIterateService messageLogIterateService; + + private final ScheduledExecutorService logCleanerExecutor; + + private final PeriodicFlushService messageLogFlushService; + private final PeriodicFlushService actionLogFlushService; + + public DefaultStorage(final BrokerRole role, final StorageConfig config, final CheckpointLoader loader) { + this.config = config; + this.consumerLogManager = new ConsumerLogManager(config); + this.messageLog = new MessageLog(config, consumerLogManager); + this.pullLogManager = new PullLogManager(config); + this.actionLog = new ActionLog(config); + + this.checkpointManager = new CheckpointManager(role, config, loader); + this.checkpointManager.fixOldVersionCheckpointIfShould(consumerLogManager, pullLogManager); + // must init after offset manager created + this.consumeQueueManager = new ConsumeQueueManager(this); + + this.pullLogFlusher = new PullLogFlusher(config, checkpointManager, pullLogManager); + this.actionEventBus = new FixedExecOrderEventBus(); + this.actionEventBus.subscribe(ActionEvent.class, new PullLogBuilder(this)); + this.actionEventBus.subscribe(ActionEvent.class, new MaxSequencesUpdater(checkpointManager)); + this.actionEventBus.subscribe(ActionEvent.class, pullLogFlusher); + this.actionLogIterateService = new ActionLogIterateService(actionLog, checkpointManager, actionEventBus); + + this.consumerLogFlusher = new ConsumerLogFlusher(config, checkpointManager, consumerLogManager); + this.messageEventBus = new FixedExecOrderEventBus(); + this.messageEventBus.subscribe(MessageLogMeta.class, new BuildConsumerLogEventListener(consumerLogManager)); + this.messageEventBus.subscribe(MessageLogMeta.class, consumerLogFlusher); + this.messageLogIterateService = new MessageLogIterateService(messageLog, checkpointManager, messageEventBus); + + this.logCleanerExecutor = Executors.newSingleThreadScheduledExecutor(new ThreadFactoryBuilder().setNameFormat("log-cleaner-%d").build()); + + this.messageLogFlushService = new PeriodicFlushService(new MessageLogFlushProvider()); + this.actionLogFlushService = new PeriodicFlushService(new ActionLogFlushProvider()); + } + + @Override + public void start() { + messageLogFlushService.start(); + actionLogFlushService.start(); + actionLogIterateService.start(); + messageLogIterateService.start(); + + messageLogIterateService.blockUntilReplayDone(); + actionLogIterateService.blockUntilReplayDone(); + // must call this after message log replay done + consumerLogManager.initConsumerLogOffset(); + + logCleanerExecutor.scheduleAtFixedRate( + new LogCleaner(), 0, config.getLogRetentionCheckIntervalSeconds(), TimeUnit.SECONDS); + } + + @Override + public StorageConfig getStorageConfig() { + return config; + } + + @Override + public void destroy() { + safeClose(actionLogIterateService); + safeClose(messageLogIterateService); + safeClose(messageLogFlushService); + safeClose(actionLogFlushService); + safeClose(consumerLogFlusher); + safeClose(pullLogFlusher); + safeClose(checkpointManager); + safeClose(messageLog); + safeClose(consumerLogManager); + safeClose(pullLogManager); + } + + private void safeClose(AutoCloseable closeable) { + if (closeable == null) return; + try { + closeable.close(); + } catch (Exception ignore) { + LOG.debug("close resource failed"); + } + } + + @Override + public synchronized PutMessageResult appendMessage(RawMessage message) { + return messageLog.putMessage(message); + } + + @Override + public SegmentBuffer getMessageData(long wroteOffset) { + return messageLog.getMessageData(wroteOffset); + } + + @Override + public GetMessageResult getMessage(String subject, long sequence) { + final MessageFilter always = entry -> true; + return pollMessages(subject, sequence, 1, always, true); + } + + @Override + public GetMessageResult pollMessages(String subject, long startSequence, int maxMessages) { + final MessageFilter always = entry -> true; + return pollMessages(subject, startSequence, maxMessages, always); + } + + @Override + public GetMessageResult pollMessages(String subject, long consumerLogSequence, int maxMessages, MessageFilter filter) { + return pollMessages(subject, consumerLogSequence, maxMessages, filter, false); + } + + private GetMessageResult pollMessages(String subject, long consumerLogSequence, int maxMessages, MessageFilter filter, boolean strictly) { + final GetMessageResult result = new GetMessageResult(); + + if (maxMessages <= 0) { + result.setNextBeginOffset(consumerLogSequence); + result.setStatus(GetMessageStatus.NO_MESSAGE); + return result; + } + + final ConsumerLog consumerLog = consumerLogManager.getConsumerLog(subject); + if (consumerLog == null) { + result.setNextBeginOffset(0); + result.setStatus(GetMessageStatus.SUBJECT_NOT_FOUND); + return result; + } + + final OffsetBound bound = consumerLog.getOffsetBound(); + final long minSequence = bound.getMinOffset(); + final long maxSequence = bound.getMaxOffset(); + result.setMinOffset(minSequence); + result.setMaxOffset(maxSequence); + + if (maxSequence == 0) { + result.setNextBeginOffset(maxSequence); + result.setStatus(GetMessageStatus.NO_MESSAGE); + return result; + } + + if (consumerLogSequence < 0) { + result.setConsumerLogRange(new OffsetRange(maxSequence, maxSequence)); + result.setNextBeginOffset(maxSequence); + result.setStatus(GetMessageStatus.SUCCESS); + return result; + } + + if (consumerLogSequence > maxSequence) { + result.setNextBeginOffset(maxSequence); + result.setStatus(GetMessageStatus.OFFSET_OVERFLOW); + return result; + } + + if (strictly && consumerLogSequence < minSequence) { + result.setNextBeginOffset(consumerLogSequence); + result.setStatus(GetMessageStatus.EMPTY_CONSUMER_LOG); + return result; + } + + final long start = consumerLogSequence < minSequence ? minSequence : consumerLogSequence; + final SegmentBuffer consumerLogBuffer = consumerLog.selectIndexBuffer(start); + if (consumerLogBuffer == null) { + result.setNextBeginOffset(start); + result.setStatus(GetMessageStatus.EMPTY_CONSUMER_LOG); + return result; + } + + if (!consumerLogBuffer.retain()) { + result.setNextBeginOffset(start); + result.setStatus(GetMessageStatus.EMPTY_CONSUMER_LOG); + return result; + } + + long nextBeginSequence = start; + try { + final int maxMessagesInBytes = maxMessages * ConsumerLog.CONSUMER_LOG_UNIT_BYTES; + for (int i = 0; i < maxMessagesInBytes; i += ConsumerLog.CONSUMER_LOG_UNIT_BYTES) { + if (i >= consumerLogBuffer.getSize()) { + break; + } + + final ConsumerLogEntry entry = ConsumerLogEntry.Factory.create(); + final ByteBuffer buffer = consumerLogBuffer.getBuffer(); + entry.setTimestamp(buffer.getLong()); + entry.setWroteOffset(buffer.getLong()); + entry.setWroteBytes(buffer.getInt()); + entry.setHeaderSize(buffer.getShort()); + + if (!filter.filter(entry)) break; + + final SegmentBuffer messageBuffer = messageLog.getMessage(entry.getWroteOffset(), entry.getWroteBytes(), entry.getHeaderSize()); + if (messageBuffer != null && messageBuffer.retain()) { + result.addSegmentBuffer(messageBuffer); + } else { + QMon.readMessageReturnNullCountInc(subject); + LOG.warn("read message log failed. consumerLogSequence: {}, wrote consumerLogSequence: {}, wrote bytes: {}, payload consumerLogSequence: {}", + nextBeginSequence, entry.getWroteOffset(), entry.getWroteBytes(), entry.getHeaderSize()); + + //如果前面已经获取到了消息,中间因为任何原因导致获取不到消息,则需要提前退出,避免consumer sequence中间出现空洞 + if (result.getSegmentBuffers().size() > 0) { + break; + } + } + nextBeginSequence += 1; + } + } finally { + consumerLogBuffer.release(); + } + result.setNextBeginOffset(nextBeginSequence); + result.setConsumerLogRange(new OffsetRange(start, nextBeginSequence - 1)); + result.setStatus(GetMessageStatus.SUCCESS); + return result; + } + + @Override + public long getMaxMessageOffset() { + return messageLog.getMaxOffset(); + } + + @Override + public long getMinMessageOffset() { + return messageLog.getMinOffset(); + } + + @Override + public long getMaxActionLogOffset() { + return actionLog.getMaxOffset(); + } + + public long getMinActionLogOffset() { + return actionLog.getMinOffset(); + } + + @Override + public long getMaxMessageSequence(String subject) { + final ConsumerLog consumerLog = consumerLogManager.getConsumerLog(subject); + if (consumerLog == null) { + return 0; + } else { + return consumerLog.nextSequence(); + } + } + + @Override + public PutMessageResult putAction(Action action) { + return actionLog.addAction(action); + } + + @Override + public List putPullLogs(String subject, String group, String consumerId, List messages) { + final PullLog pullLog = pullLogManager.getOrCreate(subject, group, consumerId); + return pullLog.putPullLogMessages(messages); + } + + @Override + public CheckpointManager getCheckpointManager() { + return checkpointManager; + } + + @Override + public ConsumerGroupProgress getConsumerGroupProgress(final String subject, final String group) { + return checkpointManager.getConsumerGroupProgress(subject, group); + } + + @Override + public Collection allConsumerGroupProgresses() { + return checkpointManager.allConsumerGroupProgresses(); + } + + @Override + public long getMaxPulledMessageSequence(String subject, String group) { + return checkpointManager.getMaxPulledMessageSequence(subject, group); + } + + @Override + public long getMessageSequenceByPullLog(String subject, String group, String consumerId, long pullLogSequence) { + final PullLog log = pullLogManager.get(subject, group, consumerId); + if (log == null) { + return -1; + } + + return log.getMessageSequence(pullLogSequence); + } + + @Override + public void disableLagMonitor(String subject, String group) { + consumeQueueManager.disableLagMonitor(subject, group); + } + + @Override + public Table allPullLogs() { + return pullLogManager.getLogs(); + } + + @Override + public void destroyPullLog(String subject, String group, String consumerId) { + if (pullLogManager.destroy(subject, group, consumerId)) { + checkpointManager.removeConsumerProgress(subject, group, consumerId); + } + } + + @Override + public ConsumeQueue locateConsumeQueue(String subject, String group) { + return consumeQueueManager.getOrCreate(subject, group); + } + + @Override + public Map locateSubjectConsumeQueues(String subject) { + return consumeQueueManager.getBySubject(subject); + } + + @Override + public void registerEventListener(final Class clazz, final FixedExecOrderEventBus.Listener listener) { + messageEventBus.subscribe(clazz, listener); + } + + @Override + public void registerActionEventListener(final FixedExecOrderEventBus.Listener listener) { + actionEventBus.subscribe(ActionEvent.class, listener); + } + + @Override + public SegmentBuffer getActionLogData(long offset) { + return actionLog.getMessageData(offset); + } + + @Override + public boolean appendMessageLogData(long startOffset, ByteBuffer data) { + return messageLog.appendData(startOffset, data); + } + + @Override + public boolean appendActionLogData(long startOffset, ByteBuffer data) { + return actionLog.appendData(startOffset, data); + } + + private class BuildConsumerLogEventListener implements FixedExecOrderEventBus.Listener { + private final ConsumerLogManager consumerLogManager; + private final Map offsets; + + private BuildConsumerLogEventListener(final ConsumerLogManager consumerLogManager) { + this.consumerLogManager = consumerLogManager; + // TODO(keli.wang): is load offset from consumer log enough? + this.offsets = new HashMap<>(consumerLogManager.currentConsumerLogOffset()); + } + + @Override + public void onEvent(final MessageLogMeta event) { + if (isFirstEventOfLogSegment(event)) { + LOG.info("first event of log segment. event: {}", event); + // TODO(keli.wang): need catch all exception here? + consumerLogManager.createOffsetFileFor(event.getBaseOffset(), offsets); + } + + updateOffset(event); + + final ConsumerLog consumerLog = consumerLogManager.getOrCreateConsumerLog(event.getSubject()); + if (consumerLog.nextSequence() != event.getSequence()) { + LOG.error("next sequence not equals to max sequence. subject: {}, received seq: {}, received offset: {}, diff: {}", + event.getSubject(), event.getSequence(), event.getWroteOffset(), event.getSequence() - consumerLog.nextSequence()); + } + final boolean success = consumerLog.putMessageLogOffset(event.getSequence(), event.getWroteOffset(), event.getWroteBytes(), event.getHeaderSize()); + checkpointManager.updateMessageReplayState(event); + messageEventBus.post(new ConsumerLogWroteEvent(event.getSubject(), success)); + } + + private boolean isFirstEventOfLogSegment(final MessageLogMeta event) { + return event.getWroteOffset() == event.getBaseOffset(); + } + + private void updateOffset(final MessageLogMeta meta) { + final String subject = meta.getSubject(); + final long sequence = meta.getSequence(); + if (offsets.containsKey(subject)) { + offsets.merge(subject, sequence, Math::max); + } else { + offsets.put(subject, sequence); + } + } + } + + private class MessageLogFlushProvider implements PeriodicFlushService.FlushProvider { + @Override + public int getInterval() { + return DEFAULT_FLUSH_INTERVAL; + } + + @Override + public void flush() { + messageLog.flush(); + } + } + + private class LogCleaner implements Runnable { + + @Override + public void run() { + try { + messageLog.clean(); + consumerLogManager.clean(); + pullLogManager.clean(allConsumerGroupProgresses()); + actionLog.clean(); + } catch (Throwable e) { + LOG.error("log cleaner caught exception.", e); + } + } + } + + private class ActionLogFlushProvider implements PeriodicFlushService.FlushProvider { + @Override + public int getInterval() { + return DEFAULT_FLUSH_INTERVAL; + } + + @Override + public void flush() { + actionLog.flush(); + } + } +} diff --git a/qmq-store/src/main/java/qunar/tc/qmq/store/FlushHook.java b/qmq-store/src/main/java/qunar/tc/qmq/store/FlushHook.java new file mode 100644 index 00000000..ca8ab6b5 --- /dev/null +++ b/qmq-store/src/main/java/qunar/tc/qmq/store/FlushHook.java @@ -0,0 +1,25 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.store; + +/** + * @author keli.wang + * @since 2018/9/28 + */ +public interface FlushHook { + void beforeFlush(); +} diff --git a/qmq-store/src/main/java/qunar/tc/qmq/store/GetMessageResult.java b/qmq-store/src/main/java/qunar/tc/qmq/store/GetMessageResult.java new file mode 100644 index 00000000..2fe4147b --- /dev/null +++ b/qmq-store/src/main/java/qunar/tc/qmq/store/GetMessageResult.java @@ -0,0 +1,119 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.store; + +import java.util.ArrayList; +import java.util.List; + +/** + * @author keli.wang + * @since 2017/7/6 + */ +public class GetMessageResult { + private final List segmentBuffers = new ArrayList<>(100); + private int bufferTotalSize = 0; + + private GetMessageStatus status; + private long minOffset; + private long maxOffset; + private long nextBeginOffset; + + private OffsetRange consumerLogRange; + + public GetMessageResult() { + } + + public GetMessageResult(GetMessageStatus status) { + this.status = status; + } + + public GetMessageStatus getStatus() { + return status; + } + + public void setStatus(GetMessageStatus status) { + this.status = status; + } + + public long getMinOffset() { + return minOffset; + } + + public void setMinOffset(long minOffset) { + this.minOffset = minOffset; + } + + public long getMaxOffset() { + return maxOffset; + } + + public void setMaxOffset(long maxOffset) { + this.maxOffset = maxOffset; + } + + public List getSegmentBuffers() { + return segmentBuffers; + } + + public void addSegmentBuffer(final SegmentBuffer segmentBuffer) { + segmentBuffers.add(segmentBuffer); + bufferTotalSize += segmentBuffer.getSize(); + } + + public int getMessageNum() { + return segmentBuffers.size(); + } + + public long getNextBeginOffset() { + return nextBeginOffset; + } + + public void setNextBeginOffset(long nextBeginOffset) { + this.nextBeginOffset = nextBeginOffset; + } + + public int getBufferTotalSize() { + return bufferTotalSize; + } + + public OffsetRange getConsumerLogRange() { + return consumerLogRange; + } + + public void setConsumerLogRange(OffsetRange consumerLogRange) { + this.consumerLogRange = consumerLogRange; + } + + public void release() { + for (SegmentBuffer buffer : segmentBuffers) { + buffer.release(); + } + } + + @Override + public String toString() { + return "GetMessageResult{" + + "segmentBuffers=" + segmentBuffers.size() + + ", bufferTotalSize=" + bufferTotalSize + + ", status=" + status + + ", minOffset=" + minOffset + + ", maxOffset=" + maxOffset + + ", nextBeginOffset=" + nextBeginOffset + + ", consumerLogRange=" + consumerLogRange + + '}'; + } +} diff --git a/qmq-store/src/main/java/qunar/tc/qmq/store/GetMessageStatus.java b/qmq-store/src/main/java/qunar/tc/qmq/store/GetMessageStatus.java new file mode 100644 index 00000000..006dc80e --- /dev/null +++ b/qmq-store/src/main/java/qunar/tc/qmq/store/GetMessageStatus.java @@ -0,0 +1,33 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.store; + +/** + * @author keli.wang + * @since 2017/7/6 + */ +public enum GetMessageStatus { + SUCCESS, + + OFFSET_OVERFLOW, + + NO_MESSAGE, + + SUBJECT_NOT_FOUND, + + EMPTY_CONSUMER_LOG +} diff --git a/qmq-store/src/main/java/qunar/tc/qmq/store/GroupAndSubject.java b/qmq-store/src/main/java/qunar/tc/qmq/store/GroupAndSubject.java new file mode 100644 index 00000000..c9b8470d --- /dev/null +++ b/qmq-store/src/main/java/qunar/tc/qmq/store/GroupAndSubject.java @@ -0,0 +1,51 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.store; + +/** + * Created by zhaohui.yu + * 8/3/18 + */ +public class GroupAndSubject { + private static final String GROUP_INDEX_DELIM = "@"; + + private final String subject; + + private final String group; + + public GroupAndSubject(String subject, String group) { + this.subject = subject; + this.group = group; + } + + public static String groupAndSubject(String subject, String group) { + return group + GROUP_INDEX_DELIM + subject; + } + + public static GroupAndSubject parse(String groupAndSubject) { + String[] arr = groupAndSubject.split(GROUP_INDEX_DELIM); + return new GroupAndSubject(arr[1], arr[0]); + } + + public String getSubject() { + return subject; + } + + public String getGroup() { + return group; + } +} diff --git a/qmq-store/src/main/java/qunar/tc/qmq/store/LogManager.java b/qmq-store/src/main/java/qunar/tc/qmq/store/LogManager.java new file mode 100644 index 00000000..1c222ff1 --- /dev/null +++ b/qmq-store/src/main/java/qunar/tc/qmq/store/LogManager.java @@ -0,0 +1,349 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.store; + +import com.google.common.base.Preconditions; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.InvalidPathException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentSkipListMap; +import java.util.function.Predicate; + +/** + * @author keli.wang + * @since 2017/7/3 + */ +public class LogManager { + private static final Logger LOG = LoggerFactory.getLogger(LogManager.class); + + private final File logDir; + private final int fileSize; + + private final StorageConfig config; + + private final LogSegmentValidator segmentValidator; + private final ConcurrentSkipListMap segments = new ConcurrentSkipListMap<>(); + + private long flushedOffset = 0; + + public LogManager(final File dir, final int fileSize, final StorageConfig config, final LogSegmentValidator segmentValidator) { + this(dir, fileSize, config, segmentValidator, false); + } + + public LogManager(final File dir, final int fileSize, final StorageConfig config, final LogSegmentValidator segmentValidator, boolean freeAfterRecover) { + this.logDir = dir; + this.fileSize = fileSize; + this.config = config; + this.segmentValidator = segmentValidator; + createAndValidateLogDir(); + loadLogs(); + recover(freeAfterRecover); + } + + // ensure dir ok + private void createAndValidateLogDir() { + if (!logDir.exists()) { + LOG.info("Log directory {} not found, try create it.", logDir.getAbsoluteFile()); + try { + Files.createDirectories(logDir.toPath()); + } catch (InvalidPathException e) { + LOG.error("log directory char array: {}", Arrays.toString(logDir.getAbsolutePath().toCharArray())); + throw new RuntimeException("Failed to create log directory " + logDir.getAbsolutePath(), e); + } catch (Exception e) { + throw new RuntimeException("Failed to create log directory " + logDir.getAbsolutePath(), e); + } + } + + if (!logDir.isDirectory() || !logDir.canRead()) { + throw new RuntimeException(logDir.getAbsolutePath() + " is not a readable log directory"); + } + } + + private void loadLogs() { + LOG.info("Loading logs."); + try { + final File[] files = logDir.listFiles(); + if (files == null) return; + + for (final File file : files) { + if (file.getName().startsWith(".")) { + continue; + } + try { + final LogSegment segment = new LogSegment(file, fileSize); + segment.setWrotePosition(fileSize); + segments.put(segment.getBaseOffset(), segment); + LOG.info("Load {} success.", file.getAbsolutePath()); + } catch (IOException e) { + LOG.error("Load {} failed.", file.getAbsolutePath()); + } + } + } finally { + LOG.info("Load logs done."); + } + } + + private void recover(boolean freeAfterRecover) { + if (segments.isEmpty()) { + return; + } + + LOG.info("Recovering logs."); + final List baseOffsets = new ArrayList<>(segments.navigableKeySet()); + final int offsetCount = baseOffsets.size(); + long offset = -1; + for (int i = offsetCount - 2; i < offsetCount; i++) { + if (i < 0) continue; + + final LogSegment segment = segments.get(baseOffsets.get(i)); + try { + offset = segment.getBaseOffset(); + + final LogSegmentValidator.ValidateResult result = segmentValidator.validate(segment); + offset += result.getValidatedSize(); + if (result.getStatus() == LogSegmentValidator.ValidateStatus.COMPLETE) { + segment.setWrotePosition(segment.getFileSize()); + } else { + break; + } + } finally { + if (freeAfterRecover) { + segment.free(); + } + } + } + flushedOffset = offset; + + final long maxOffset = latestSegment().getBaseOffset() + latestSegment().getFileSize(); + final int relativeOffset = (int) (offset % fileSize); + final LogSegment segment = locateSegment(offset); + if (segment != null && maxOffset != offset) { + segment.setWrotePosition(relativeOffset); + LOG.info("recover wrote offset to {}:{}", segment, segment.getWrotePosition()); + // TODO(keli.wang): should delete crash file + } + LOG.info("Recover done."); + } + + public LogSegment locateSegment(final long offset) { + if (isBaseOffset(offset)) { + return segments.get(offset); + } + + final Map.Entry entry = segments.lowerEntry(offset); + if (entry == null) { + return null; + } else { + return entry.getValue(); + } + } + + private boolean isBaseOffset(final long offset) { + return offset % fileSize == 0; + } + + public LogSegment firstSegment() { + final Map.Entry entry = segments.firstEntry(); + return entry == null ? null : entry.getValue(); + } + + public LogSegment latestSegment() { + final Map.Entry entry = segments.lastEntry(); + return entry == null ? null : entry.getValue(); + } + + public LogSegment allocNextSegment() { + final long nextBaseOffset = nextSegmentBaseOffset(); + return allocSegment(nextBaseOffset); + } + + private long nextSegmentBaseOffset() { + final LogSegment segment = latestSegment(); + if (segment == null) { + return 0; + } else { + return segment.getBaseOffset() + fileSize; + } + } + + private LogSegment allocSegment(final long baseOffset) { + final File nextSegmentFile = new File(logDir, StoreUtils.offset2FileName(baseOffset)); + try { + final LogSegment segment = new LogSegment(nextSegmentFile, fileSize); + segments.put(baseOffset, segment); + LOG.info("alloc new segment file {}", segment); + return segment; + } catch (IOException e) { + LOG.error("Failed create new segment file. file: {}", nextSegmentFile.getAbsolutePath()); + } + return null; + } + + public LogSegment allocOrResetSegments(final long expectedOffset) { + final long baseOffset = computeBaseOffset(expectedOffset); + + if (segments.isEmpty()) { + return allocSegment(baseOffset); + } + + if (nextSegmentBaseOffset() == baseOffset && latestSegment().isFull()) { + return allocSegment(baseOffset); + } + + LOG.warn("All segments are too old, need to delete all segment now. Current base offset: {}, expect base offset: {}", + latestSegment().getBaseOffset(), baseOffset); + deleteAllSegments(); + + return allocSegment(baseOffset); + } + + private long computeBaseOffset(final long offset) { + return offset - (offset % fileSize); + } + + private void deleteAllSegments() { + for (Map.Entry entry : segments.entrySet()) { + deleteSegment(entry.getKey(), entry.getValue()); + } + } + + public long getMinOffset() { + final LogSegment segment = firstSegment(); + if (segment == null) { + return 0; + } + return segment.getBaseOffset(); + } + + public long getMaxOffset() { + final LogSegment segment = latestSegment(); + if (segment == null) { + return 0; + } + return segment.getBaseOffset() + segment.getWrotePosition(); + } + + public boolean flush() { + boolean result = true; + final LogSegment segment = locateSegment(flushedOffset); + if (segment != null) { + final int offset = segment.flush(); + final long where = segment.getBaseOffset() + offset; + result = where == this.flushedOffset; + this.flushedOffset = where; + } + return result; + } + + public void close() { + for (final LogSegment segment : segments.values()) { + segment.close(); + } + } + + public void deleteExpiredSegments(final long retentionMs) { + deleteExpiredSegments(retentionMs, null); + } + + public void deleteExpiredSegments(final long retentionMs, DeleteHook afterDeleted) { + final long deleteUntil = System.currentTimeMillis() - retentionMs; + Preconditions.checkState(deleteUntil > 0, "retentionMs不应该超过当前时间"); + + Predicate predicate = segment -> { + if (!config.isDeleteExpiredLogsEnable()) { + LOG.info("should delete expired segment {}, but delete expired logs is disabled for now", segment); + return false; + } + return segment.getLastModifiedTime() < deleteUntil; + }; + deleteSegments(predicate, afterDeleted); + } + + public void deleteSegmentsBeforeOffset(final long offset) { + if (offset == -1) return; + Predicate predicate = segment -> segment.getBaseOffset() < offset; + deleteSegments(predicate, null); + } + + public void deleteSegments(Predicate predicate, DeleteHook afterDeleted) { + int count = segments.size(); + if (count <= 1) return; + + for (final Map.Entry entry : segments.entrySet()) { + if (count <= 1) return; + + final LogSegment segment = entry.getValue(); + + if (predicate.test(segment)) { + if (deleteSegment(entry.getKey(), segment)) { + count = count - 1; + executeHook(afterDeleted, segment); + LOG.info("remove expired segment success. segment: {}", segment); + } else { + LOG.warn("remove expired segment failed. segment: {}", segment); + return; + } + } + } + } + + private void executeHook(DeleteHook hook, LogSegment segment) { + if (hook == null) return; + + hook.afterDeleted(segment); + } + + private boolean deleteSegment(final long key, final LogSegment segment) { + if (!segment.release()) return false; + segments.remove(key); + segment.destroy(); + return true; + } + + public void destroy() { + deleteAllSegments(); + logDir.delete(); + } + + public interface DeleteHook { + void afterDeleted(LogSegment segment); + } + + public boolean clean(Long key) { + LogSegment segment = segments.get(key); + if (null == segment){ + LOG.error("clean message segment log error,segment:{} is null",key); + return false; + } + + if (deleteSegment(key, segment)) { + LOG.info("remove expired segment success. segment: {}", segment); + return true; + } else { + LOG.warn("remove expired segment failed. segment: {}", segment); + return false; + } + } +} diff --git a/qmq-store/src/main/java/qunar/tc/qmq/store/LogSegment.java b/qmq-store/src/main/java/qunar/tc/qmq/store/LogSegment.java new file mode 100644 index 00000000..02edd8f3 --- /dev/null +++ b/qmq-store/src/main/java/qunar/tc/qmq/store/LogSegment.java @@ -0,0 +1,229 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.store; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import sun.misc.Cleaner; +import sun.nio.ch.DirectBuffer; + +import java.io.File; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.io.RandomAccessFile; +import java.nio.ByteBuffer; +import java.nio.MappedByteBuffer; +import java.nio.channels.FileChannel; +import java.util.concurrent.atomic.AtomicInteger; + +/** + * @author keli.wang + * @since 2017/7/3 + */ +public class LogSegment extends ReferenceObject { + private static final Logger LOG = LoggerFactory.getLogger(LogSegment.class); + + private final File file; + private final int fileSize; + private final String fileName; + private final long baseOffset; + + private final AtomicInteger wrotePosition = new AtomicInteger(0); + + private RandomAccessFile rawFile; + private FileChannel fileChannel; + private MappedByteBuffer mappedByteBuffer; + + public LogSegment(final File file, final int fileSize) throws IOException { + this.file = file; + this.fileSize = fileSize; + this.fileName = file.getAbsolutePath(); + this.baseOffset = Long.parseLong(file.getName()); + + boolean success = false; + try { + rawFile = new RandomAccessFile(file, "rw"); + fileChannel = rawFile.getChannel(); + mappedByteBuffer = fileChannel.map(FileChannel.MapMode.READ_WRITE, 0, fileSize); + success = true; + } catch (FileNotFoundException e) { + LOG.error("create file channel failed. file: {}", fileName, e); + throw e; + } catch (IOException e) { + LOG.error("map file failed. file: {}", fileName, e); + throw e; + } finally { + if (!success && fileChannel != null) { + fileChannel.close(); + } + } + } + + public AppendMessageResult append(final T message, final MessageAppender appender) { + final int currentPos = wrotePosition.get(); + if (currentPos > fileSize) { + return new AppendMessageResult<>(AppendMessageStatus.UNKNOWN_ERROR); + } + if (currentPos == fileSize) { + return new AppendMessageResult<>(AppendMessageStatus.END_OF_FILE); + } + + final ByteBuffer buffer = mappedByteBuffer.slice(); + buffer.position(currentPos); + final AppendMessageResult result = appender.doAppend(getBaseOffset(), buffer, fileSize - currentPos, message); + this.wrotePosition.addAndGet(result.getWroteBytes()); + return result; + } + + public boolean appendData(final ByteBuffer data) { + final int currentPos = wrotePosition.get(); + final int size = data.limit(); + if (currentPos + size > fileSize) { + return false; + } + + try { + fileChannel.position(currentPos); + fileChannel.write(data); + } catch (Throwable e) { + LOG.error("Append data to log segment failed.", e); + } + this.wrotePosition.addAndGet(size); + return true; + } + + public void fillPreBlank(final ByteBuffer blank, final long untilWhere) { + final int currentPos = wrotePosition.get(); + final int untilPos = (int) (untilWhere % fileSize); + final int size = untilPos - currentPos; + if (size <= 0) return; + + final ByteBuffer buffer = mappedByteBuffer.slice(); + buffer.position(currentPos); + buffer.limit(untilPos); + buffer.put(blank); + this.wrotePosition.addAndGet(size); + } + + public int getWrotePosition() { + return wrotePosition.get(); + } + + public void setWrotePosition(int position) { + wrotePosition.set(position); + } + + public int getFileSize() { + return fileSize; + } + + public long getLastModifiedTime() { + return file.lastModified(); + } + + public long getBaseOffset() { + return baseOffset; + } + + public boolean isFull() { + return wrotePosition.get() == fileSize; + } + + public ByteBuffer sliceByteBuffer() { + return mappedByteBuffer.slice(); + } + + public SegmentBuffer selectSegmentBuffer(final int pos) { + final int wrotePosition = getWrotePosition(); + if (pos < wrotePosition && pos >= 0) { + + final ByteBuffer buffer = mappedByteBuffer.slice(); + buffer.position(pos); + + final ByteBuffer bufferNew = buffer.slice(); + final int size = wrotePosition - pos; + bufferNew.limit(size); + return new SegmentBuffer(getBaseOffset() + pos, bufferNew, size, this); + } + + return null; + } + + public SegmentBuffer selectSegmentBuffer(final int pos, final int size) { + final int wrotePosition = getWrotePosition(); + if ((pos + size) > wrotePosition) { + return null; + } + + final ByteBuffer buffer = mappedByteBuffer.slice(); + buffer.position(pos); + + final ByteBuffer bufferNew = buffer.slice(); + bufferNew.limit(size); + return new SegmentBuffer(getBaseOffset() + pos, bufferNew, size, this); + } + + public int flush() { + final int value = wrotePosition.get(); + try { + if (fileChannel.position() != 0) { + fileChannel.force(false); + } else { + mappedByteBuffer.force(); + } + } catch (IOException e) { + throw new RuntimeException("flush segment failed. segment: " + fileName, e); + } + + return value; + } + + public void close() { + try { + fileChannel.close(); + clean(mappedByteBuffer); + } catch (Exception e) { + LOG.error("close file channel failed. file: {}", fileName, e); + } + } + + // MappedByteBuffer 是不会随着 channel 的关闭而释放的,需要通过特殊方法调用 clean 来释放 + private void clean(final ByteBuffer buffer) { + if (buffer == null || !buffer.isDirect() || buffer.capacity() == 0) { + return; + } + + final Cleaner cleaner = ((DirectBuffer) buffer).cleaner(); + cleaner.clean(); + } + + public boolean destroy() { + close(); + return file.delete(); + } + + @Override + public String toString() { + return "LogSegment{" + + "file=" + file.getAbsolutePath() + + '}'; + } + + public void free() { + MmapUtil.free(rawFile); + } +} diff --git a/qmq-store/src/main/java/qunar/tc/qmq/store/LogSegmentValidator.java b/qmq-store/src/main/java/qunar/tc/qmq/store/LogSegmentValidator.java new file mode 100644 index 00000000..530a9a9e --- /dev/null +++ b/qmq-store/src/main/java/qunar/tc/qmq/store/LogSegmentValidator.java @@ -0,0 +1,48 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.store; + +/** + * @author keli.wang + * @since 2017/7/5 + */ +public interface LogSegmentValidator { + enum ValidateStatus { + COMPLETE, + PARTIAL + } + + class ValidateResult { + private final ValidateStatus status; + private final int validatedSize; + + public ValidateResult(ValidateStatus status, int validatedSize) { + this.status = status; + this.validatedSize = validatedSize; + } + + public ValidateStatus getStatus() { + return status; + } + + public int getValidatedSize() { + return validatedSize; + } + } + + ValidateResult validate(final LogSegment segment); +} diff --git a/qmq-store/src/main/java/qunar/tc/qmq/store/MagicCode.java b/qmq-store/src/main/java/qunar/tc/qmq/store/MagicCode.java new file mode 100644 index 00000000..bb4927ed --- /dev/null +++ b/qmq-store/src/main/java/qunar/tc/qmq/store/MagicCode.java @@ -0,0 +1,37 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.store; + +/** + * @author keli.wang + * @since 2017/7/6 + */ +public final class MagicCode { + public static final int CONSUMER_LOG_MAGIC_V1 = 0xA3B2C100; + + public static final int MESSAGE_LOG_MAGIC_V1 = 0xA1B2C300; + + // add crc field for message body + public static final int MESSAGE_LOG_MAGIC_V2 = 0xA1B2C301; + + //add tags field for message header + public static final int MESSAGE_LOG_MAGIC_V3 = 0xA1B2C302; + + public static final int PULL_LOG_MAGIC_V1 = 0xC1B2A300; + + public static final int ACTION_LOG_MAGIC_V1 = 0xC3B2A100; +} diff --git a/qmq-store/src/main/java/qunar/tc/qmq/store/MagicCodeSupport.java b/qmq-store/src/main/java/qunar/tc/qmq/store/MagicCodeSupport.java new file mode 100644 index 00000000..c0adf060 --- /dev/null +++ b/qmq-store/src/main/java/qunar/tc/qmq/store/MagicCodeSupport.java @@ -0,0 +1,31 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.store; + +/** + * @author keli.wang + * @since 2018/6/25 + */ +public final class MagicCodeSupport { + private MagicCodeSupport() { + } + + public static boolean isValidMessageLogMagicCode(final int magicCode) { + return magicCode == MagicCode.MESSAGE_LOG_MAGIC_V1 + || magicCode == MagicCode.MESSAGE_LOG_MAGIC_V2 || magicCode == MagicCode.MESSAGE_LOG_MAGIC_V3; + } +} diff --git a/qmq-store/src/main/java/qunar/tc/qmq/store/MaxAckedPullLogSequence.java b/qmq-store/src/main/java/qunar/tc/qmq/store/MaxAckedPullLogSequence.java new file mode 100644 index 00000000..a9b142b9 --- /dev/null +++ b/qmq-store/src/main/java/qunar/tc/qmq/store/MaxAckedPullLogSequence.java @@ -0,0 +1,75 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.store; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; + +import java.util.concurrent.atomic.AtomicLong; + +/** + * @author keli.wang + * @since 2017/8/21 + */ +public class MaxAckedPullLogSequence { + private final String subject; + private final String group; + private final String consumerId; + + private final AtomicLong maxSequence; + + @JsonCreator + public MaxAckedPullLogSequence(@JsonProperty("subject") String subject, + @JsonProperty("group") String group, + @JsonProperty("consumerId") String consumerId, + @JsonProperty("maxSequence") long maxSequence) { + this.subject = subject; + this.group = group; + this.consumerId = consumerId; + this.maxSequence = new AtomicLong(maxSequence); + } + + public String getSubject() { + return subject; + } + + public String getGroup() { + return group; + } + + public String getConsumerId() { + return consumerId; + } + + public long getMaxSequence() { + return maxSequence.get(); + } + + public void setMaxSequence(final long maxSequence) { + this.maxSequence.set(maxSequence); + } + + @Override + public String toString() { + return "MaxAckedPullLogSequence{" + + "subject='" + subject + '\'' + + ", group='" + group + '\'' + + ", consumerId='" + consumerId + '\'' + + ", maxSequence=" + maxSequence + + '}'; + } +} diff --git a/qmq-store/src/main/java/qunar/tc/qmq/store/MaxPulledMessageSequence.java b/qmq-store/src/main/java/qunar/tc/qmq/store/MaxPulledMessageSequence.java new file mode 100644 index 00000000..1176430d --- /dev/null +++ b/qmq-store/src/main/java/qunar/tc/qmq/store/MaxPulledMessageSequence.java @@ -0,0 +1,67 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.store; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; + +import java.util.concurrent.atomic.AtomicLong; + +/** + * @author keli.wang + * @since 2017/8/21 + */ +public class MaxPulledMessageSequence { + private final String subject; + private final String group; + + private final AtomicLong maxSequence; + + @JsonCreator + public MaxPulledMessageSequence(@JsonProperty("subject") String subject, + @JsonProperty("group") String group, + @JsonProperty("maxSequence") long maxSequence) { + this.subject = subject; + this.group = group; + this.maxSequence = new AtomicLong(maxSequence); + } + + public String getSubject() { + return subject; + } + + public String getGroup() { + return group; + } + + public long getMaxSequence() { + return maxSequence.get(); + } + + public void setMaxSequence(final long maxSequence) { + this.maxSequence.set(maxSequence); + } + + @Override + public String toString() { + return "MaxPulledMessageSequence{" + + "subject='" + subject + '\'' + + ", group='" + group + '\'' + + ", maxSequence=" + maxSequence + + '}'; + } +} diff --git a/qmq-store/src/main/java/qunar/tc/qmq/store/MessageAppender.java b/qmq-store/src/main/java/qunar/tc/qmq/store/MessageAppender.java new file mode 100644 index 00000000..fba87523 --- /dev/null +++ b/qmq-store/src/main/java/qunar/tc/qmq/store/MessageAppender.java @@ -0,0 +1,27 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.store; + +import java.nio.ByteBuffer; + +/** + * @author keli.wang + * @since 2017/7/5 + */ +public interface MessageAppender { + AppendMessageResult doAppend(final long baseOffset, final ByteBuffer targetBuffer, final int freeSpace, final T message); +} diff --git a/qmq-store/src/main/java/qunar/tc/qmq/store/MessageCheckpoint.java b/qmq-store/src/main/java/qunar/tc/qmq/store/MessageCheckpoint.java new file mode 100644 index 00000000..561ab6af --- /dev/null +++ b/qmq-store/src/main/java/qunar/tc/qmq/store/MessageCheckpoint.java @@ -0,0 +1,61 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.store; + +import java.util.Map; + +/** + * @author keli.wang + * @since 2018/9/12 + */ +public class MessageCheckpoint { + private final Map maxSequences; + // mark this object is read from old version snapshot file + // TODO(keli.wang): delete this after all broker group is migrate to new snapshot format + private boolean fromOldVersion; + private long offset; + + public MessageCheckpoint(long offset, Map maxSequences) { + this(false, offset, maxSequences); + } + + public MessageCheckpoint(boolean fromOldVersion, long offset, Map maxSequences) { + this.fromOldVersion = fromOldVersion; + this.offset = offset; + this.maxSequences = maxSequences; + } + + public boolean isFromOldVersion() { + return fromOldVersion; + } + + public void setFromOldVersion(boolean fromOldVersion) { + this.fromOldVersion = fromOldVersion; + } + + public long getOffset() { + return offset; + } + + public void setOffset(long offset) { + this.offset = offset; + } + + public Map getMaxSequences() { + return maxSequences; + } +} diff --git a/qmq-store/src/main/java/qunar/tc/qmq/store/MessageCheckpointSerde.java b/qmq-store/src/main/java/qunar/tc/qmq/store/MessageCheckpointSerde.java new file mode 100644 index 00000000..13dda450 --- /dev/null +++ b/qmq-store/src/main/java/qunar/tc/qmq/store/MessageCheckpointSerde.java @@ -0,0 +1,87 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.store; + +import com.google.common.base.Charsets; +import com.google.common.base.Joiner; +import com.google.common.base.Splitter; +import com.google.common.base.Strings; +import com.google.common.io.LineReader; + +import java.io.IOException; +import java.io.StringReader; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * @author keli.wang + * @since 2018/9/12 + */ +public class MessageCheckpointSerde implements Serde { + private static final int VERSION_V1 = 1; + private static final int VERSION_V2 = 2; + + private static final char NEWLINE = '\n'; + private static final Joiner SLASH_JOINER = Joiner.on('/'); + private static final Splitter SLASH_SPLITTER = Splitter.on('/'); + + @Override + public byte[] toBytes(final MessageCheckpoint state) { + final StringBuilder data = new StringBuilder(); + data.append(VERSION_V2).append(NEWLINE); + data.append(state.getOffset()).append(NEWLINE); + state.getMaxSequences().forEach((subject, sequence) -> data.append(SLASH_JOINER.join(subject, sequence)).append(NEWLINE)); + return data.toString().getBytes(Charsets.UTF_8); + } + + @Override + public MessageCheckpoint fromBytes(final byte[] data) { + try { + final LineReader reader = new LineReader(new StringReader(new String(data, Charsets.UTF_8))); + final int version = Integer.parseInt(reader.readLine()); + switch (version) { + case VERSION_V1: + return new MessageCheckpoint(true, -1, new HashMap<>()); + case VERSION_V2: + return parseV2(reader); + default: + throw new RuntimeException("unknown snapshot content version " + version); + } + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + private MessageCheckpoint parseV2(final LineReader reader) throws IOException { + final long offset = Long.parseLong(reader.readLine()); + final Map sequences = new HashMap<>(); + while (true) { + final String line = reader.readLine(); + if (Strings.isNullOrEmpty(line)) { + break; + } + + final List parts = SLASH_SPLITTER.splitToList(line); + final String subject = parts.get(0); + final long maxSequence = Long.parseLong(parts.get(1)); + sequences.put(subject, maxSequence); + } + + return new MessageCheckpoint(offset, sequences); + } +} diff --git a/qmq-store/src/main/java/qunar/tc/qmq/store/MessageFilter.java b/qmq-store/src/main/java/qunar/tc/qmq/store/MessageFilter.java new file mode 100644 index 00000000..d736b0d8 --- /dev/null +++ b/qmq-store/src/main/java/qunar/tc/qmq/store/MessageFilter.java @@ -0,0 +1,25 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.store; + +/** + * Created by zhaohui.yu + * 9/3/18 + */ +interface MessageFilter { + boolean filter(ConsumerLogEntry entry); +} diff --git a/qmq-store/src/main/java/qunar/tc/qmq/store/MessageLog.java b/qmq-store/src/main/java/qunar/tc/qmq/store/MessageLog.java new file mode 100644 index 00000000..1ad5d313 --- /dev/null +++ b/qmq-store/src/main/java/qunar/tc/qmq/store/MessageLog.java @@ -0,0 +1,282 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.store; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import qunar.tc.qmq.base.RawMessage; +import qunar.tc.qmq.monitor.QMon; +import qunar.tc.qmq.utils.Crc32; + +import java.io.File; +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; + +/** + * @author keli.wang + * @since 2017/7/4 + */ +public class MessageLog implements AutoCloseable { + private static final Logger LOG = LoggerFactory.getLogger(MessageLog.class); + + public static final int PER_SEGMENT_FILE_SIZE = 100 * 1024 * 1024; + + //4 bytes magic code + 1 byte attribute + 8 bytes timestamp + public static final int MIN_RECORD_BYTES = 13; + + private final StorageConfig config; + private final ConsumerLogManager consumerLogManager; + private final LogManager logManager; + private final MessageAppender messageAppender = new RawMessageAppender(); + + public MessageLog(final StorageConfig config, final ConsumerLogManager consumerLogManager) { + this.config = config; + this.consumerLogManager = consumerLogManager; + this.logManager = new LogManager(new File(config.getMessageLogStorePath()), PER_SEGMENT_FILE_SIZE, config, new MessageLogSegmentValidator()); + consumerLogManager.adjustConsumerLogMinOffset(logManager.firstSegment()); + } + + private static int recordSize(final int subjectSize, final int payloadSize) { + return 4 // magic code + + 1 // attributes + + 8 // timestamp + + 8 // message logical offset + + 2 // subject size + + (subjectSize > 0 ? subjectSize : 0) + + 8 // payload crc32 + + 4 // payload size + + (payloadSize > 0 ? payloadSize : 0); + } + + public long getMaxOffset() { + return logManager.getMaxOffset(); + } + + public long getMinOffset() { + return logManager.getMinOffset(); + } + + public PutMessageResult putMessage(final RawMessage message) { + final AppendMessageResult result; + LogSegment segment = logManager.latestSegment(); + if (segment == null) { + segment = logManager.allocNextSegment(); + } + + if (segment == null) { + return new PutMessageResult(PutMessageStatus.CREATE_MAPPED_FILE_FAILED, null); + } + + result = segment.append(message, messageAppender); + switch (result.getStatus()) { + case SUCCESS: + break; + case END_OF_FILE: + LogSegment logSegment = logManager.allocNextSegment(); + if (logSegment == null) { + return new PutMessageResult(PutMessageStatus.CREATE_MAPPED_FILE_FAILED, null); + } + return putMessage(message); + case MESSAGE_SIZE_EXCEEDED: + return new PutMessageResult(PutMessageStatus.MESSAGE_ILLEGAL, result); + default: + return new PutMessageResult(PutMessageStatus.UNKNOWN_ERROR, result); + } + + return new PutMessageResult(PutMessageStatus.SUCCESS, result); + } + + public SegmentBuffer getMessage(final long wroteOffset, final int wroteBytes, final short headerSize) { + long payloadOffset = wroteOffset + headerSize; + final LogSegment segment = logManager.locateSegment(payloadOffset); + if (segment == null) return null; + + final int payloadSize = wroteBytes - headerSize; + final int pos = (int) (payloadOffset % PER_SEGMENT_FILE_SIZE); + final SegmentBuffer result = segment.selectSegmentBuffer(pos, payloadSize); + if (result == null) return null; + + result.setWroteOffset(wroteOffset); + return result; + } + + public SegmentBuffer getMessageData(final long offset) { + final LogSegment segment = logManager.locateSegment(offset); + if (segment == null) return null; + + final int pos = (int) (offset % PER_SEGMENT_FILE_SIZE); + return segment.selectSegmentBuffer(pos); + } + + public boolean appendData(final long startOffset, final ByteBuffer data) { + LogSegment segment = logManager.locateSegment(startOffset); + if (segment == null) { + segment = logManager.allocOrResetSegments(startOffset); + fillPreBlank(segment, startOffset); + } + + return segment.appendData(data); + } + + private void fillPreBlank(LogSegment segment, long untilWhere) { + final ByteBuffer buffer = ByteBuffer.allocate(17); + buffer.putInt(MagicCode.MESSAGE_LOG_MAGIC_V3); + buffer.put((byte) 2); + buffer.putLong(System.currentTimeMillis()); + buffer.putInt((int) (untilWhere % PER_SEGMENT_FILE_SIZE)); + segment.fillPreBlank(buffer, untilWhere); + } + + public void flush() { + final long start = System.currentTimeMillis(); + try { + logManager.flush(); + } finally { + QMon.flushMessageLogTimer(System.currentTimeMillis() - start); + } + } + + @Override + public void close() { + logManager.close(); + } + + public void clean() { + logManager.deleteExpiredSegments(config.getMessageLogRetentionMs(), segment -> { + consumerLogManager.adjustConsumerLogMinOffset(logManager.firstSegment()); + + final String fileName = StoreUtils.offsetFileNameForSegment(segment); + final String path = config.getMessageLogStorePath(); + final File file = new File(path, fileName); + try { + if (!file.delete()) { + LOG.warn("delete offset file failed. file: {}", fileName); + } + } catch (Exception e) { + LOG.warn("delete offset file failed.. file: {}", fileName, e); + } + }); + } + + public MessageLogMetaVisitor newVisitor(long iterateFrom) { + return new MessageLogMetaVisitor(logManager, iterateFrom); + } + + private static class MessageLogSegmentValidator implements LogSegmentValidator { + @Override + public ValidateResult validate(LogSegment segment) { + final int fileSize = segment.getFileSize(); + final ByteBuffer buffer = segment.sliceByteBuffer(); + + int position = 0; + while (true) { + if (position == fileSize) { + return new ValidateResult(ValidateStatus.COMPLETE, fileSize); + } + + final int result = consumeAndValidateMessage(segment, buffer); + if (result == -1) { + return new ValidateResult(ValidateStatus.PARTIAL, position); + } else if (result == 0) { + return new ValidateResult(ValidateStatus.COMPLETE, fileSize); + } else { + position += result; + } + } + } + + private int consumeAndValidateMessage(final LogSegment segment, final ByteBuffer buffer) { + final int magic = buffer.getInt(); + if (!MagicCodeSupport.isValidMessageLogMagicCode(magic)) { + return -1; + } + + final byte attributes = buffer.get(); + buffer.getLong(); + if (attributes == 2) { + return buffer.getInt(); + } else if (attributes == 1) { + return 0; + } else if (attributes == 0) { + buffer.getLong(); + final short subjectSize = buffer.getShort(); + buffer.position(buffer.position() + subjectSize); + final long crc = buffer.getLong(); + final int payloadSize = buffer.getInt(); + final byte[] payload = new byte[payloadSize]; + buffer.get(payload); + final long computedCrc = Crc32.crc32(payload); + if (computedCrc == crc) { + return recordSize(subjectSize, payloadSize); + } else { + LOG.warn("crc check failed. stored crc: {}, computed crc: {}, segment: {}", crc, computedCrc, segment); + return -1; + } + } else { + return -1; + } + } + } + + private class RawMessageAppender implements MessageAppender { + private static final int MAX_BYTES = 1024 * 1024 * 50; // 50M + + private static final byte ATTR_EMPTY_RECORD = 1; + private static final byte ATTR_MESSAGE_RECORD = 0; + + private final ByteBuffer workingBuffer = ByteBuffer.allocate(MAX_BYTES); + + @Override + public AppendMessageResult doAppend(long baseOffset, ByteBuffer targetBuffer, int freeSpace, RawMessage message) { + workingBuffer.clear(); + + final String subject = message.getHeader().getSubject(); + final byte[] subjectBytes = subject.getBytes(StandardCharsets.UTF_8); + + final long wroteOffset = baseOffset + targetBuffer.position(); + final int recordSize = recordSize(subjectBytes.length, message.getBodySize()); + + if (recordSize != freeSpace && recordSize + MIN_RECORD_BYTES > freeSpace) { + workingBuffer.limit(freeSpace); + workingBuffer.putInt(MagicCode.MESSAGE_LOG_MAGIC_V3); + workingBuffer.put(ATTR_EMPTY_RECORD); + workingBuffer.putLong(System.currentTimeMillis()); + targetBuffer.put(workingBuffer.array(), 0, freeSpace); + return new AppendMessageResult<>(AppendMessageStatus.END_OF_FILE, wroteOffset, freeSpace, null); + } else { + final long sequence = consumerLogManager.getOffsetOrDefault(subject, 0); + + workingBuffer.limit(recordSize); + workingBuffer.putInt(MagicCode.MESSAGE_LOG_MAGIC_V3); + workingBuffer.put(ATTR_MESSAGE_RECORD); + workingBuffer.putLong(System.currentTimeMillis()); + workingBuffer.putLong(sequence); + workingBuffer.putShort((short) subjectBytes.length); + workingBuffer.put(subjectBytes); + workingBuffer.putLong(message.getHeader().getBodyCrc()); + workingBuffer.putInt(message.getBodySize()); + workingBuffer.put(message.getBody().nioBuffer()); + targetBuffer.put(workingBuffer.array(), 0, recordSize); + + consumerLogManager.incOffset(subject); + + final long payloadOffset = wroteOffset + recordSize - message.getBodySize(); + return new AppendMessageResult<>(AppendMessageStatus.SUCCESS, wroteOffset, recordSize, new MessageSequence(sequence, payloadOffset)); + } + } + } +} diff --git a/qmq-store/src/main/java/qunar/tc/qmq/store/MessageLogIterateService.java b/qmq-store/src/main/java/qunar/tc/qmq/store/MessageLogIterateService.java new file mode 100644 index 00000000..2acda8b4 --- /dev/null +++ b/qmq-store/src/main/java/qunar/tc/qmq/store/MessageLogIterateService.java @@ -0,0 +1,133 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.store; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import qunar.tc.qmq.monitor.QMon; +import qunar.tc.qmq.store.event.FixedExecOrderEventBus; + +import java.util.Optional; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.LongAdder; + +/** + * @author keli.wang + * @since 2017/8/23 + */ +public class MessageLogIterateService implements AutoCloseable { + private static final Logger LOG = LoggerFactory.getLogger(MessageLogIterateService.class); + + private final MessageLog log; + private final FixedExecOrderEventBus dispatcher; + private final Thread dispatcherThread; + + private final LongAdder iterateFrom; + private volatile boolean stop = false; + + public MessageLogIterateService(final MessageLog log, final CheckpointManager checkpointManager, final FixedExecOrderEventBus dispatcher) { + this.log = log; + this.dispatcher = dispatcher; + this.dispatcherThread = new Thread(new Dispatcher()); + this.dispatcherThread.setName("MessageLogIterator"); + this.iterateFrom = new LongAdder(); + this.iterateFrom.add(initialMessageIterateFrom(log, checkpointManager)); + + QMon.replayMessageLogLag(() -> (double) replayMessageLogLag()); + } + + private long initialMessageIterateFrom(final MessageLog log, final CheckpointManager checkpointManager) { + if (checkpointManager.getMessageCheckpointOffset() <= 0) { + return log.getMaxOffset(); + } + if (checkpointManager.getMessageCheckpointOffset() > log.getMaxOffset()) { + return log.getMaxOffset(); + } + return checkpointManager.getMessageCheckpointOffset(); + } + + public void start() { + dispatcherThread.start(); + } + + public void blockUntilReplayDone() { + LOG.info("replay message log initial lag: {}; min: {}, max: {}, from: {}", + replayMessageLogLag(), log.getMinOffset(), log.getMaxOffset(), iterateFrom.longValue()); + + while (replayMessageLogLag() > 0) { + LOG.info("waiting replay message log ..."); + try { + TimeUnit.SECONDS.sleep(1); + } catch (InterruptedException e) { + LOG.warn("block until replay done interrupted", e); + } + } + } + + private long replayMessageLogLag() { + return log.getMaxOffset() - iterateFrom.longValue(); + } + + @Override + public void close() { + stop = true; + try { + dispatcherThread.join(); + } catch (InterruptedException e) { + LOG.error("action log dispatcher thread interrupted", e); + } + } + + private class Dispatcher implements Runnable { + @Override + public void run() { + while (!stop) { + try { + processLog(); + } catch (Throwable e) { + QMon.replayMessageLogFailedCountInc(); + LOG.error("replay message log failed, will retry.", e); + } + } + } + + private void processLog() { + final long startOffset = iterateFrom.longValue(); + final MessageLogMetaVisitor visitor = log.newVisitor(startOffset); + if (startOffset != visitor.getStartOffset()) { + iterateFrom.reset(); + iterateFrom.add(visitor.getStartOffset()); + } + + while (true) { + final Optional meta = visitor.nextRecord(); + if (meta == null) { + break; + } + + meta.ifPresent(dispatcher::post); + } + iterateFrom.add(visitor.visitedBufferSize()); + + try { + TimeUnit.MILLISECONDS.sleep(5); + } catch (InterruptedException e) { + LOG.warn("action log dispatcher sleep interrupted"); + } + } + } +} diff --git a/qmq-store/src/main/java/qunar/tc/qmq/store/MessageLogMeta.java b/qmq-store/src/main/java/qunar/tc/qmq/store/MessageLogMeta.java new file mode 100644 index 00000000..cdf4d20c --- /dev/null +++ b/qmq-store/src/main/java/qunar/tc/qmq/store/MessageLogMeta.java @@ -0,0 +1,76 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.store; + +/** + * @author keli.wang + * @since 2017/8/23 + */ +public class MessageLogMeta { + private final String subject; + private final long sequence; + private final long wroteOffset; + private final int wroteBytes; + private final short headerSize; + private final long baseOffset; + + public MessageLogMeta(String subject, long sequence, long wroteOffset, int wroteBytes, short headerSize, long baseOffset) { + this.subject = subject; + this.sequence = sequence; + this.wroteOffset = wroteOffset; + this.wroteBytes = wroteBytes; + this.headerSize = headerSize; + this.baseOffset = baseOffset; + } + + public String getSubject() { + return subject; + } + + public long getSequence() { + return sequence; + } + + + public long getWroteOffset() { + return wroteOffset; + } + + public int getWroteBytes() { + return wroteBytes; + } + + public short getHeaderSize() { + return headerSize; + } + + public long getBaseOffset() { + return baseOffset; + } + + @Override + public String toString() { + return "MessageLogMeta{" + + "subject='" + subject + '\'' + + ", sequence=" + sequence + + ", wroteOffset=" + wroteOffset + + ", wroteBytes=" + wroteBytes + + ", headerSize=" + headerSize + + ", baseOffset=" + baseOffset + + '}'; + } +} diff --git a/qmq-store/src/main/java/qunar/tc/qmq/store/MessageLogMetaVisitor.java b/qmq-store/src/main/java/qunar/tc/qmq/store/MessageLogMetaVisitor.java new file mode 100644 index 00000000..7c0e8f28 --- /dev/null +++ b/qmq-store/src/main/java/qunar/tc/qmq/store/MessageLogMetaVisitor.java @@ -0,0 +1,144 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.store; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; +import java.util.Optional; +import java.util.concurrent.atomic.AtomicInteger; + +/** + * @author keli.wang + * @since 2017/8/23 + */ +public class MessageLogMetaVisitor { + private static final Logger LOG = LoggerFactory.getLogger(MessageLogMetaVisitor.class); + + private final LogSegment currentSegment; + private final SegmentBuffer currentBuffer; + private final AtomicInteger visitedBufferSize = new AtomicInteger(0); + private final long startOffset; + + public MessageLogMetaVisitor(final LogManager logManager, final long startOffset) { + final long initialOffset = initialOffset(logManager, startOffset); + this.currentSegment = logManager.locateSegment(initialOffset); + this.currentBuffer = selectBuffer(initialOffset); + this.startOffset = initialOffset; + } + + private long initialOffset(final LogManager logManager, final long originStart) { + if (originStart < logManager.getMinOffset()) { + LOG.error("initial message log visitor offset less than min offset. start: {}, min: {}", + originStart, logManager.getMinOffset()); + return logManager.getMinOffset(); + } + + return originStart; + } + + public long getStartOffset() { + return startOffset; + } + + public Optional nextRecord() { + if (currentBuffer == null) { + return null; + } + + return readOneRecord(currentBuffer.getBuffer()); + } + + public int visitedBufferSize() { + if (currentBuffer == null) { + return 0; + } + return visitedBufferSize.get(); + } + + private SegmentBuffer selectBuffer(final long startOffset) { + if (currentSegment == null) { + return null; + } + final int pos = (int) (startOffset % currentSegment.getFileSize()); + return currentSegment.selectSegmentBuffer(pos); + } + + private Optional readOneRecord(final ByteBuffer buffer) { + if (buffer.remaining() < MessageLog.MIN_RECORD_BYTES) { + return null; + } + + final int startPos = buffer.position(); + final long wroteOffset = startOffset + startPos; + final int magic = buffer.getInt(); + if (!MagicCodeSupport.isValidMessageLogMagicCode(magic)) { + visitedBufferSize.set(currentBuffer.getSize()); + return null; + } + + final byte attributes = buffer.get(); + //skip timestamp + buffer.getLong(); + if (attributes == 2) { + if (buffer.remaining() < Integer.BYTES) { + return null; + } + final int blankSize = buffer.getInt(); + visitedBufferSize.addAndGet(blankSize + (buffer.position() - startPos)); + return Optional.empty(); + } else if (attributes == 1) { + visitedBufferSize.set(currentBuffer.getSize()); + return null; + } else { + if (buffer.remaining() < (Long.BYTES + Integer.BYTES)) { + return null; + } + final long sequence = buffer.getLong(); + + final short subjectSize = buffer.getShort(); + if (buffer.remaining() < subjectSize) { + return null; + } + final byte[] subjectBytes = new byte[subjectSize]; + buffer.get(subjectBytes); + + if (buffer.remaining() < Long.BYTES) { + return null; + } + //skip crc + buffer.getLong(); + + if (buffer.remaining() < Integer.BYTES) { + return null; + } + final int payloadSize = buffer.getInt(); + if (buffer.remaining() < payloadSize) { + return null; + } + + final short headerSize = (short) (buffer.position() - startPos); + buffer.position(buffer.position() + payloadSize); + final int wroteBytes = buffer.position() - startPos; + visitedBufferSize.addAndGet(wroteBytes); + return Optional.of(new MessageLogMeta(new String(subjectBytes, StandardCharsets.UTF_8), + sequence, wroteOffset, wroteBytes, headerSize, currentSegment.getBaseOffset())); + } + } +} diff --git a/qmq-store/src/main/java/qunar/tc/qmq/store/MessageSequence.java b/qmq-store/src/main/java/qunar/tc/qmq/store/MessageSequence.java new file mode 100644 index 00000000..1ced9cd4 --- /dev/null +++ b/qmq-store/src/main/java/qunar/tc/qmq/store/MessageSequence.java @@ -0,0 +1,39 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.store; + +/** + * @author keli.wang + * @since 2017/8/19 + */ +public class MessageSequence { + private final long sequence; + private final long physicalOffset; + + public MessageSequence(long sequence, long physicalOffset) { + this.sequence = sequence; + this.physicalOffset = physicalOffset; + } + + public long getSequence() { + return sequence; + } + + public long getPhysicalOffset() { + return physicalOffset; + } +} diff --git a/qmq-store/src/main/java/qunar/tc/qmq/store/MmapUtil.java b/qmq-store/src/main/java/qunar/tc/qmq/store/MmapUtil.java new file mode 100644 index 00000000..bbbce148 --- /dev/null +++ b/qmq-store/src/main/java/qunar/tc/qmq/store/MmapUtil.java @@ -0,0 +1,60 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.store; + +import io.netty.util.internal.NativeLibraryLoader; +import io.netty.util.internal.PlatformDependent; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.FileDescriptor; +import java.io.RandomAccessFile; +import java.lang.reflect.Field; + +public class MmapUtil { + private static final Logger LOGGER = LoggerFactory.getLogger(MmapUtil.class); + + private static Field fdField; + + private static boolean inited = true; + + static { + try { + NativeLibraryLoader.load("mmaputil", PlatformDependent.getClassLoader(MmapUtil.class)); + Class clazz = FileDescriptor.class; + fdField = clazz.getDeclaredField("fd"); + fdField.setAccessible(true); + } catch (Exception e) { + inited = false; + LOGGER.error("init failed", e); + } + } + + public static void free(RandomAccessFile file) { + if (!inited) return; + + try { + FileDescriptor fd = file.getFD(); + int intFd = (Integer) fdField.get(fd); + free0(intFd, 0, file.length()); + } catch (Exception e) { + LOGGER.error("free error", e); + } + } + + private native static void free0(int fd, long offset, long length); +} diff --git a/qmq-store/src/main/java/qunar/tc/qmq/store/OffsetBound.java b/qmq-store/src/main/java/qunar/tc/qmq/store/OffsetBound.java new file mode 100644 index 00000000..c598375b --- /dev/null +++ b/qmq-store/src/main/java/qunar/tc/qmq/store/OffsetBound.java @@ -0,0 +1,39 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.store; + +/** + * @author keli.wang + * @since 2018/7/6 + */ +class OffsetBound { + private final long minOffset; + private final long maxOffset; + + OffsetBound(long minOffset, long maxOffset) { + this.minOffset = minOffset; + this.maxOffset = maxOffset; + } + + public long getMinOffset() { + return minOffset; + } + + public long getMaxOffset() { + return maxOffset; + } +} diff --git a/qmq-store/src/main/java/qunar/tc/qmq/store/OffsetRange.java b/qmq-store/src/main/java/qunar/tc/qmq/store/OffsetRange.java new file mode 100644 index 00000000..a06543e2 --- /dev/null +++ b/qmq-store/src/main/java/qunar/tc/qmq/store/OffsetRange.java @@ -0,0 +1,47 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.store; + +/** + * @author keli.wang + * @since 2017/8/8 + */ +public class OffsetRange { + private final long begin; + private final long end; + + public OffsetRange(long begin, long end) { + this.begin = begin; + this.end = end; + } + + public long getBegin() { + return begin; + } + + public long getEnd() { + return end; + } + + @Override + public String toString() { + return "OffsetRange{" + + "begin=" + begin + + ", end=" + end + + '}'; + } +} diff --git a/qmq-store/src/main/java/qunar/tc/qmq/store/PeriodicFlushService.java b/qmq-store/src/main/java/qunar/tc/qmq/store/PeriodicFlushService.java new file mode 100644 index 00000000..b8fa34c2 --- /dev/null +++ b/qmq-store/src/main/java/qunar/tc/qmq/store/PeriodicFlushService.java @@ -0,0 +1,83 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.store; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import qunar.tc.qmq.concurrent.NamedThreadFactory; + +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ScheduledFuture; +import java.util.concurrent.TimeUnit; + +/** + * @author keli.wang + * @since 2017/7/7 + */ +public class PeriodicFlushService implements AutoCloseable { + private static final Logger LOG = LoggerFactory.getLogger(PeriodicFlushService.class); + + private final String name; + private final FlushProvider flushProvider; + private final ScheduledExecutorService scheduler; + private volatile ScheduledFuture future; + + public PeriodicFlushService(final FlushProvider flushProvider) { + this.name = flushProvider.getClass().getSimpleName(); + this.flushProvider = flushProvider; + this.scheduler = Executors.newSingleThreadScheduledExecutor(new NamedThreadFactory(name)); + } + + public void start() { + future = scheduler.scheduleWithFixedDelay( + new FlushRunnable(), + flushProvider.getInterval(), + flushProvider.getInterval(), + TimeUnit.MILLISECONDS); + } + + @Override + public void close() { + try { + if (future != null) { + future.cancel(false); + } + LOG.info("will flush one more time for {} before shutdown flush service.", name); + flushProvider.flush(); + } catch (Exception e) { + LOG.error("shutdown flush service for {} failed.", name, e); + } + } + + public interface FlushProvider { + int getInterval(); + + void flush(); + } + + private class FlushRunnable implements Runnable { + @Override + public void run() { + try { + flushProvider.flush(); + } catch (Throwable e) { + LOG.error("flushProvider {} flush failed.", name, e); + } + } + } +} diff --git a/qmq-store/src/main/java/qunar/tc/qmq/store/PullLog.java b/qmq-store/src/main/java/qunar/tc/qmq/store/PullLog.java new file mode 100644 index 00000000..58381435 --- /dev/null +++ b/qmq-store/src/main/java/qunar/tc/qmq/store/PullLog.java @@ -0,0 +1,191 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.store; + +import qunar.tc.qmq.monitor.QMon; + +import java.io.File; +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.List; + +/** + * @author keli.wang + * @since 2017/8/2 + */ +public class PullLog { + private static final int PULL_LOG_UNIT_BYTES = 8; // 8 bytes message sequence + private static final int PULL_LOG_SIZE = PULL_LOG_UNIT_BYTES * 10_000_000; // TODO(keli.wang): to config + + private final StorageConfig config; + private final LogManager logManager; + private final MessageAppender messageAppender = new PullLogMessageAppender(); + + public PullLog(final StorageConfig config, final String consumerId, final String groupAndSubject) { + this.config = config; + this.logManager = new LogManager(buildPullLogPath(consumerId, groupAndSubject), PULL_LOG_SIZE, config, new PullLogSegmentValidator(), true); + } + + private File buildPullLogPath(final String consumerId, final String groupAndSubject) { + return new File(new File(config.getPullLogStorePath(), consumerId), groupAndSubject); + } + + public synchronized List putPullLogMessages(final List messages) { + final List results = new ArrayList<>(messages.size()); + for (final PullLogMessage message : messages) { + results.add(directPutMessage(message)); + } + return results; + } + + private PutMessageResult directPutMessage(final PullLogMessage message) { + final long sequence = message.getSequence(); + + if (sequence < getMaxOffset()) { + return new PutMessageResult(PutMessageStatus.ALREADY_WRITTEN, null); + } + + final long expectPhysicalOffset = sequence * PULL_LOG_UNIT_BYTES; + LogSegment segment = logManager.locateSegment(expectPhysicalOffset); + if (segment == null) { + segment = logManager.allocOrResetSegments(expectPhysicalOffset); + } + fillPreBlank(segment, expectPhysicalOffset); + + final AppendMessageResult result = segment.append(message, messageAppender); + switch (result.getStatus()) { + case SUCCESS: + break; + case END_OF_FILE: + logManager.allocNextSegment(); + return directPutMessage(message); + default: + return new PutMessageResult(PutMessageStatus.UNKNOWN_ERROR, result); + } + + return new PutMessageResult(PutMessageStatus.SUCCESS, result); + } + + private void fillPreBlank(final LogSegment segment, final long untilWhere) { + final PullLogMessage blankMessage = new PullLogMessage(0, -1); + final long startOffset = segment.getBaseOffset() + segment.getWrotePosition(); + for (long i = startOffset; i < untilWhere; i += PULL_LOG_UNIT_BYTES) { + segment.append(blankMessage, messageAppender); + } + } + + public long getMessageSequence(long pullLogSequence) { + final SegmentBuffer result = selectIndexBuffer(pullLogSequence); + if (result == null) { + return -1; + } + + if (!result.retain()) { + return -1; + } + + try { + final ByteBuffer buffer = result.getBuffer(); + buffer.getInt(); + return buffer.getLong(); + } finally { + result.release(); + } + } + + private SegmentBuffer selectIndexBuffer(final long startIndex) { + final long startOffset = startIndex * PULL_LOG_UNIT_BYTES; + final LogSegment segment = logManager.locateSegment(startOffset); + if (segment == null) { + return null; + } else { + return segment.selectSegmentBuffer((int) (startOffset % PULL_LOG_SIZE)); + } + } + + public long getMinOffset() { + return logManager.getMinOffset() / PULL_LOG_UNIT_BYTES; + } + + public long getMaxOffset() { + return logManager.getMaxOffset() / PULL_LOG_UNIT_BYTES; + } + + public void flush() { + logManager.flush(); + QMon.flushPullLogCountInc(); + } + + public void close() { + logManager.close(); + } + + public void clean(long sequence) { + long offset = sequence * PULL_LOG_UNIT_BYTES; + logManager.deleteSegmentsBeforeOffset(offset); + } + + public void destroy() { + logManager.destroy(); + } + + + private static class PullLogMessageAppender implements MessageAppender { + private final ByteBuffer workingBuffer = ByteBuffer.allocate(PULL_LOG_UNIT_BYTES); + + @Override + public AppendMessageResult doAppend(long baseOffset, ByteBuffer targetBuffer, int freeSpace, PullLogMessage message) { + workingBuffer.clear(); + + final long wroteOffset = baseOffset + targetBuffer.position(); + workingBuffer.flip(); + workingBuffer.limit(PULL_LOG_UNIT_BYTES); + workingBuffer.putLong(message.getMessageSequence()); + targetBuffer.put(workingBuffer.array(), 0, PULL_LOG_UNIT_BYTES); + + final long messageIndex = wroteOffset / PULL_LOG_UNIT_BYTES; + return new AppendMessageResult<>(AppendMessageStatus.SUCCESS, wroteOffset, PULL_LOG_UNIT_BYTES, new MessageSequence(messageIndex, wroteOffset)); + } + } + + private static class PullLogSegmentValidator implements LogSegmentValidator { + @Override + public ValidateResult validate(LogSegment segment) { + final int fileSize = segment.getFileSize(); + final ByteBuffer buffer = segment.sliceByteBuffer(); + + int position = 0; + while (true) { + if (position == fileSize) { + return new ValidateResult(ValidateStatus.COMPLETE, fileSize); + } + + final int result = consumeAndValidateMessage(buffer); + if (result == -1) { + return new ValidateResult(ValidateStatus.PARTIAL, position); + } else { + position += result; + } + } + } + + private int consumeAndValidateMessage(final ByteBuffer buffer) { + buffer.getLong(); + return PULL_LOG_UNIT_BYTES; + } + } +} diff --git a/qmq-store/src/main/java/qunar/tc/qmq/store/PullLogFlusher.java b/qmq-store/src/main/java/qunar/tc/qmq/store/PullLogFlusher.java new file mode 100644 index 00000000..70b16a8e --- /dev/null +++ b/qmq-store/src/main/java/qunar/tc/qmq/store/PullLogFlusher.java @@ -0,0 +1,110 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.store; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import qunar.tc.qmq.concurrent.NamedThreadFactory; +import qunar.tc.qmq.monitor.QMon; +import qunar.tc.qmq.store.action.ActionEvent; +import qunar.tc.qmq.store.event.FixedExecOrderEventBus; + +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicLong; + +/** + * @author keli.wang + * @since 2018/9/12 + */ +public class PullLogFlusher implements FixedExecOrderEventBus.Listener, AutoCloseable { + private static final Logger LOG = LoggerFactory.getLogger(PullLogFlusher.class); + + private final StorageConfig config; + private final CheckpointManager checkpointManager; + private final PullLogManager pullLogManager; + private final ScheduledExecutorService flushExecutor; + private final AtomicLong counter; + private volatile long latestFlushTime; + + public PullLogFlusher(final StorageConfig config, final CheckpointManager checkpointManager, final PullLogManager pullLogManager) { + this.config = config; + this.checkpointManager = checkpointManager; + this.pullLogManager = pullLogManager; + this.flushExecutor = Executors.newSingleThreadScheduledExecutor(new NamedThreadFactory("pull-log-flusher")); + this.counter = new AtomicLong(0); + this.latestFlushTime = -1; + + scheduleForceFlushTask(); + } + + private void scheduleForceFlushTask() { + flushExecutor.scheduleWithFixedDelay(this::tryForceSubmitFlushTask, 1, 1, TimeUnit.MINUTES); + } + + private void tryForceSubmitFlushTask() { + final long interval = System.currentTimeMillis() - latestFlushTime; + if (interval < TimeUnit.MINUTES.toMillis(1)) { + return; + } + + submitFlushTask(); + } + + @Override + public void onEvent(ActionEvent event) { + final long count = counter.incrementAndGet(); + if (count < config.getActionCheckpointInterval()) { + return; + } + + QMon.pullLogFlusherExceedCheckpointIntervalCountInc(); + submitFlushTask(); + } + + private synchronized void submitFlushTask() { + counter.set(0); + latestFlushTime = System.currentTimeMillis(); + + final Snapshot snapshot = checkpointManager.createActionCheckpointSnapshot(); + flushExecutor.submit(() -> { + final long start = System.currentTimeMillis(); + try { + pullLogManager.flush(); + checkpointManager.saveActionCheckpointSnapshot(snapshot); + } catch (Exception e) { + QMon.pullLogFlusherFlushFailedCountInc(); + LOG.error("flush pull log failed.", e); + } finally { + QMon.pullLogFlusherElapsedPerExecute(System.currentTimeMillis() - start); + } + }); + } + + @Override + public void close() { + LOG.info("try flush one more time before exit."); + submitFlushTask(); + flushExecutor.shutdown(); + try { + flushExecutor.awaitTermination(1, TimeUnit.MINUTES); + } catch (InterruptedException e) { + LOG.warn("interrupted during closing pull log flusher."); + } + } +} diff --git a/qmq-store/src/main/java/qunar/tc/qmq/store/PullLogManager.java b/qmq-store/src/main/java/qunar/tc/qmq/store/PullLogManager.java new file mode 100644 index 00000000..b094e5d4 --- /dev/null +++ b/qmq-store/src/main/java/qunar/tc/qmq/store/PullLogManager.java @@ -0,0 +1,154 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.store; + +import com.google.common.collect.HashBasedTable; +import com.google.common.collect.Table; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import qunar.tc.qmq.monitor.QMon; + +import java.io.File; +import java.util.Collection; +import java.util.Map; + +import static qunar.tc.qmq.store.GroupAndSubject.groupAndSubject; + +/** + * @author keli.wang + * @since 2017/8/3 + */ +public class PullLogManager implements AutoCloseable { + private static final Logger LOG = LoggerFactory.getLogger(PullLogManager.class); + + private final StorageConfig config; + private final Table logs; + + public PullLogManager(final StorageConfig config) { + this.config = config; + this.logs = HashBasedTable.create(); + + loadPullLogs(); + } + + private void loadPullLogs() { + final File pullLogsRoot = new File(config.getPullLogStorePath()); + final File[] consumerIdDirs = pullLogsRoot.listFiles(); + if (consumerIdDirs != null) { + for (final File consumerIdDir : consumerIdDirs) { + if (!consumerIdDir.isDirectory()) { + continue; + } + loadPullLogsByConsumerId(consumerIdDir); + } + } + } + + private void loadPullLogsByConsumerId(final File consumerIdDir) { + final File[] groupAndSubjectDirs = consumerIdDir.listFiles(); + if (groupAndSubjectDirs != null) { + for (final File groupAndSubjectDir : groupAndSubjectDirs) { + if (!groupAndSubjectDir.isDirectory()) { + continue; + } + final File[] segments = groupAndSubjectDir.listFiles(); + if (segments == null || segments.length == 0) { + LOG.info("need delete empty pull log dir: {}", groupAndSubjectDir.getAbsolutePath()); + continue; + } + + final String consumerId = consumerIdDir.getName(); + final String groupAndSubject = groupAndSubjectDir.getName(); + logs.put(consumerId, groupAndSubject, new PullLog(config, consumerId, groupAndSubject)); + } + } + } + + public PullLog get(final String subject, final String group, final String consumerId) { + String groupAndSubject = groupAndSubject(subject, group); + synchronized (logs) { + return logs.get(consumerId, groupAndSubject); + } + } + + public PullLog getOrCreate(final String subject, final String group, final String consumerId) { + final String groupAndSubject = groupAndSubject(subject, group); + synchronized (logs) { + if (!logs.contains(consumerId, groupAndSubject)) { + logs.put(consumerId, groupAndSubject, new PullLog(config, consumerId, groupAndSubject)); + } + + return logs.get(consumerId, groupAndSubject); + } + } + + public Table getLogs() { + synchronized (logs) { + return HashBasedTable.create(logs); + } + } + + public void flush() { + final long start = System.currentTimeMillis(); + try { + for (final PullLog log : logs.values()) { + log.flush(); + } + } finally { + QMon.flushPullLogTimer(System.currentTimeMillis() - start); + } + } + + public void clean(Collection progresses) { + for (ConsumerGroupProgress progress : progresses) { + final Map consumers = progress.getConsumers(); + if (consumers == null || consumers.isEmpty()) { + continue; + } + + for (final ConsumerProgress consumer : consumers.values()) { + PullLog pullLog = get(consumer.getSubject(), consumer.getGroup(), consumer.getConsumerId()); + if (pullLog == null) continue; + + pullLog.clean(consumer.getAck()); + } + } + } + + public boolean destroy(String subject, String group, String consumerId) { + PullLog pullLog = get(subject, group, consumerId); + if (pullLog == null) return false; + + pullLog.destroy(); + remove(subject, group, consumerId); + return true; + } + + private void remove(String subject, String group, String consumerId) { + String groupAndSubject = groupAndSubject(subject, group); + synchronized (logs) { + logs.remove(consumerId, groupAndSubject); + } + } + + @Override + public void close() { + for (final PullLog log : logs.values()) { + log.close(); + } + } +} diff --git a/qmq-store/src/main/java/qunar/tc/qmq/store/PullLogMessage.java b/qmq-store/src/main/java/qunar/tc/qmq/store/PullLogMessage.java new file mode 100644 index 00000000..8516cc17 --- /dev/null +++ b/qmq-store/src/main/java/qunar/tc/qmq/store/PullLogMessage.java @@ -0,0 +1,47 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.store; + +/** + * @author keli.wang + * @since 2017/8/1 + */ +public class PullLogMessage { + private final long sequence; + private final long messageSequence; + + public PullLogMessage(final long sequence, final long messageSequence) { + this.sequence = sequence; + this.messageSequence = messageSequence; + } + + public long getSequence() { + return sequence; + } + + public long getMessageSequence() { + return messageSequence; + } + + @Override + public String toString() { + return "PullLogMessage{" + + "sequence=" + sequence + + ", messageSequence=" + messageSequence + + '}'; + } +} diff --git a/qmq-store/src/main/java/qunar/tc/qmq/store/PutMessageResult.java b/qmq-store/src/main/java/qunar/tc/qmq/store/PutMessageResult.java new file mode 100644 index 00000000..8b1e821f --- /dev/null +++ b/qmq-store/src/main/java/qunar/tc/qmq/store/PutMessageResult.java @@ -0,0 +1,39 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.store; + +/** + * @author keli.wang + * @since 2017/7/4 + */ +public class PutMessageResult { + private final PutMessageStatus status; + private final AppendMessageResult result; + + public PutMessageResult(PutMessageStatus status, AppendMessageResult result) { + this.status = status; + this.result = result; + } + + public PutMessageStatus getStatus() { + return status; + } + + public AppendMessageResult getResult() { + return result; + } +} diff --git a/qmq-store/src/main/java/qunar/tc/qmq/store/PutMessageStatus.java b/qmq-store/src/main/java/qunar/tc/qmq/store/PutMessageStatus.java new file mode 100644 index 00000000..cfb1dc4a --- /dev/null +++ b/qmq-store/src/main/java/qunar/tc/qmq/store/PutMessageStatus.java @@ -0,0 +1,39 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.store; + +/** + * @author keli.wang + * @since 2017/7/5 + */ +public enum PutMessageStatus { + SUCCESS(0), + CREATE_MAPPED_FILE_FAILED(1), + MESSAGE_ILLEGAL(2), + ALREADY_WRITTEN(3), + UNKNOWN_ERROR(-1); + + private int code; + + PutMessageStatus(int code) { + this.code = code; + } + + public int getCode() { + return code; + } +} \ No newline at end of file diff --git a/qmq-store/src/main/java/qunar/tc/qmq/store/ReferenceObject.java b/qmq-store/src/main/java/qunar/tc/qmq/store/ReferenceObject.java new file mode 100644 index 00000000..49adfc6d --- /dev/null +++ b/qmq-store/src/main/java/qunar/tc/qmq/store/ReferenceObject.java @@ -0,0 +1,72 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.store; + +import io.netty.util.internal.PlatformDependent; +import qunar.tc.qmq.monitor.QMon; + +import java.util.concurrent.atomic.AtomicIntegerFieldUpdater; + +/** + * Created by zhaohui.yu + * 4/11/18 + */ +abstract class ReferenceObject { + private static final AtomicIntegerFieldUpdater REF_CNT_UPDATER; + + static { + AtomicIntegerFieldUpdater updater = + PlatformDependent.newAtomicIntegerFieldUpdater(ReferenceObject.class, "refCnt"); + if (updater == null) { + updater = AtomicIntegerFieldUpdater.newUpdater(ReferenceObject.class, "refCnt"); + } + REF_CNT_UPDATER = updater; + } + + private volatile int refCnt = 1; + + public boolean retain() { + for (; ; ) { + final int refCnt = this.refCnt; + if (refCnt < 1) { + return false; + } + + if (REF_CNT_UPDATER.compareAndSet(this, refCnt, refCnt + 1)) { + QMon.logSegmentTotalRefCountInc(); + return true; + } + } + } + + public boolean release() { + for (; ; ) { + final int refCnt = this.refCnt; + if (refCnt < 1) { + return true; + } + + if (REF_CNT_UPDATER.compareAndSet(this, refCnt, refCnt - 1)) { + final boolean allRefReleased = refCnt == 1; + if (!allRefReleased) { + QMon.logSegmentTotalRefCountDec(); + } + return allRefReleased; + } + } + } +} diff --git a/qmq-store/src/main/java/qunar/tc/qmq/store/SegmentBuffer.java b/qmq-store/src/main/java/qunar/tc/qmq/store/SegmentBuffer.java new file mode 100644 index 00000000..73c73240 --- /dev/null +++ b/qmq-store/src/main/java/qunar/tc/qmq/store/SegmentBuffer.java @@ -0,0 +1,72 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.store; + +import java.nio.ByteBuffer; + +/** + * @author keli.wang + * @since 2017/7/6 + */ +public class SegmentBuffer { + private final long startOffset; + private final int size; + + private final ByteBuffer buffer; + private final LogSegment logSegment; + + private long wroteOffset; + + public SegmentBuffer(long startOffset, ByteBuffer buffer, int size, LogSegment logSegment) { + this.startOffset = startOffset; + this.size = size; + this.buffer = buffer; + this.logSegment = logSegment; + } + + public long getStartOffset() { + return startOffset; + } + + public ByteBuffer getBuffer() { + return buffer; + } + + public int getSize() { + return size; + } + + public LogSegment getLogSegment() { + return logSegment; + } + + public long getWroteOffset() { + return wroteOffset; + } + + public void setWroteOffset(long wroteOffset) { + this.wroteOffset = wroteOffset; + } + + public boolean release() { + return logSegment.release(); + } + + public boolean retain() { + return logSegment.retain(); + } +} diff --git a/qmq-store/src/main/java/qunar/tc/qmq/store/Serde.java b/qmq-store/src/main/java/qunar/tc/qmq/store/Serde.java new file mode 100644 index 00000000..91d23cfb --- /dev/null +++ b/qmq-store/src/main/java/qunar/tc/qmq/store/Serde.java @@ -0,0 +1,27 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.store; + +/** + * @author keli.wang + * @since 2018/9/10 + */ +public interface Serde { + byte[] toBytes(final V value); + + V fromBytes(final byte[] data); +} diff --git a/qmq-store/src/main/java/qunar/tc/qmq/store/Snapshot.java b/qmq-store/src/main/java/qunar/tc/qmq/store/Snapshot.java new file mode 100644 index 00000000..5f39b5c7 --- /dev/null +++ b/qmq-store/src/main/java/qunar/tc/qmq/store/Snapshot.java @@ -0,0 +1,46 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.store; + +/** + * @author keli.wang + * @since 2018/9/10 + */ +public class Snapshot { + private final long version; + private final T data; + + public Snapshot(final long version, final T data) { + this.version = version; + this.data = data; + } + + public long getVersion() { + return version; + } + + public T getData() { + return data; + } + + @Override + public String toString() { + return "Snapshot{" + + "version=" + version + + '}'; + } +} diff --git a/qmq-store/src/main/java/qunar/tc/qmq/store/SnapshotStore.java b/qmq-store/src/main/java/qunar/tc/qmq/store/SnapshotStore.java new file mode 100644 index 00000000..c9ca5550 --- /dev/null +++ b/qmq-store/src/main/java/qunar/tc/qmq/store/SnapshotStore.java @@ -0,0 +1,178 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.store; + +import com.google.common.io.Files; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import qunar.tc.qmq.concurrent.NamedThreadFactory; + +import java.io.File; +import java.io.IOException; +import java.util.Map; +import java.util.UUID; +import java.util.concurrent.ConcurrentSkipListMap; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; + +/** + * @author keli.wang + * @since 2018/9/10 + */ +public class SnapshotStore { + private static final Logger LOG = LoggerFactory.getLogger(SnapshotStore.class); + + private final String prefix; + private final StorageConfig config; + private final File storePath; + private final Serde serde; + private final ConcurrentSkipListMap> snapshots; + private final ScheduledExecutorService cleanerExecutor; + + public SnapshotStore(final String name, final StorageConfig config, final Serde serde) { + this.prefix = name + "."; + this.config = config; + this.storePath = new File(config.getCheckpointStorePath()); + this.serde = serde; + this.snapshots = new ConcurrentSkipListMap<>(); + this.cleanerExecutor = Executors.newSingleThreadScheduledExecutor(new NamedThreadFactory(name + "-snapshot-cleaner")); + + ensureStorePath(); + loadAllSnapshots(); + scheduleSnapshotCleaner(); + } + + private void ensureStorePath() { + if (storePath.exists()) { + return; + } + + final boolean success = storePath.mkdirs(); + if (!success) { + throw new RuntimeException("failed create directory " + storePath); + } + LOG.info("create snapshot store directory {} success.", storePath); + } + + private void loadAllSnapshots() { + for (final File file : scanAllSnapshotFiles()) { + loadSnapshotFile(file); + } + } + + private File[] scanAllSnapshotFiles() { + final File[] files = storePath.listFiles((dir, name) -> name.startsWith(prefix)); + if (files == null) { + return new File[0]; + } else { + return files; + } + } + + private void loadSnapshotFile(final File file) { + try { + final long version = parseSnapshotVersion(file.getName()); + final byte[] data = Files.toByteArray(file); + final Snapshot snapshot = new Snapshot<>(version, serde.fromBytes(data)); + snapshots.put(version, snapshot); + LOG.info("load snapshot file {} success.", file.getAbsolutePath()); + } catch (Exception e) { + LOG.error("load snapshot file {} failed.", file.getAbsolutePath(), e); + } + } + + private long parseSnapshotVersion(final String filename) { + final int beginIndex = prefix.length(); + return Long.parseLong(filename.substring(beginIndex)); + } + + private void scheduleSnapshotCleaner() { + cleanerExecutor.scheduleAtFixedRate(this::tryCleanExpiredSnapshot, 1, 1, TimeUnit.MINUTES); + } + + private void tryCleanExpiredSnapshot() { + try { + while (true) { + if (snapshots.size() <= config.getCheckpointRetainCount()) { + return; + } + + removeOldestSnapshot(); + } + } catch (Throwable e) { + LOG.error("try clean expired snapshot file failed.", e); + } + } + + private void removeOldestSnapshot() { + final Map.Entry> firstEntry = snapshots.pollFirstEntry(); + final File snapshotFile = getSnapshotFile(firstEntry.getValue()); + final boolean success = snapshotFile.delete(); + if (success) { + LOG.debug("delete snapshot file {} success.", snapshotFile.getAbsolutePath()); + } else { + LOG.warn("delete snapshot file {} failed.", snapshotFile.getAbsolutePath()); + } + } + + public Snapshot latestSnapshot() { + final Map.Entry> entry = snapshots.lastEntry(); + return entry == null ? null : entry.getValue(); + } + + public synchronized void saveSnapshot(final Snapshot snapshot) { + if (snapshots.containsKey(snapshot.getVersion())) { + return; + } + + final File tmpFile = tmpFile(); + try { + Files.write(serde.toBytes(snapshot.getData()), tmpFile); + } catch (IOException e) { + LOG.error("write data into tmp snapshot file failed. file: {}", tmpFile, e); + throw new RuntimeException("write snapshot data failed.", e); + } + + final File snapshotFile = getSnapshotFile(snapshot); + if (!tmpFile.renameTo(snapshotFile)) { + LOG.error("Move tmp as snapshot file failed. tmp: {}, snapshot: {}", tmpFile, snapshotFile); + throw new RuntimeException("Move tmp as snapshot file failed."); + } + + snapshots.put(snapshot.getVersion(), snapshot); + } + + private File tmpFile() { + final String uuid = UUID.randomUUID().toString(); + return new File(storePath, uuid + ".tmp"); + } + + private File getSnapshotFile(final Snapshot snapshot) { + final String filename = prefix + StoreUtils.offset2FileName(snapshot.getVersion()); + return new File(storePath, filename); + } + + public void close() { + cleanerExecutor.shutdown(); + try { + cleanerExecutor.awaitTermination(1, TimeUnit.MINUTES); + } catch (InterruptedException e) { + LOG.warn("interrupted during shutdown cleaner executor with prefix {}", prefix); + } + } +} diff --git a/qmq-store/src/main/java/qunar/tc/qmq/store/Storage.java b/qmq-store/src/main/java/qunar/tc/qmq/store/Storage.java new file mode 100644 index 00000000..67ba2812 --- /dev/null +++ b/qmq-store/src/main/java/qunar/tc/qmq/store/Storage.java @@ -0,0 +1,92 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.store; + +import com.google.common.collect.Table; +import qunar.tc.qmq.base.RawMessage; +import qunar.tc.qmq.common.Disposable; +import qunar.tc.qmq.store.action.ActionEvent; +import qunar.tc.qmq.store.event.FixedExecOrderEventBus; + +import java.nio.ByteBuffer; +import java.util.Collection; +import java.util.List; +import java.util.Map; + +/** + * @author keli.wang + * @since 2017/7/4 + */ +public interface Storage extends Disposable { + void start(); + + StorageConfig getStorageConfig(); + + PutMessageResult appendMessage(final RawMessage message); + + SegmentBuffer getMessageData(final long wroteOffset); + + GetMessageResult getMessage(String subject, long sequence); + + GetMessageResult pollMessages(String subject, long startSequence, int maxMessages); + + GetMessageResult pollMessages(final String subject, final long startSequence, final int maxMessages, MessageFilter filter); + + long getMaxMessageOffset(); + + long getMinMessageOffset(); + + long getMaxActionLogOffset(); + + long getMinActionLogOffset(); + + long getMaxMessageSequence(final String subject); + + PutMessageResult putAction(final Action action); + + List putPullLogs(final String subject, final String group, final String consumerId, final List messages); + + CheckpointManager getCheckpointManager(); + + ConsumerGroupProgress getConsumerGroupProgress(final String subject, final String group); + + Collection allConsumerGroupProgresses(); + + long getMaxPulledMessageSequence(String subject, String group); + + long getMessageSequenceByPullLog(final String subject, final String group, final String consumerId, final long pullLogSequence); + + ConsumeQueue locateConsumeQueue(final String subject, final String group); + + Map locateSubjectConsumeQueues(final String subject); + + void registerEventListener(final Class clazz, final FixedExecOrderEventBus.Listener listener); + + void registerActionEventListener(final FixedExecOrderEventBus.Listener listener); + + SegmentBuffer getActionLogData(final long offset); + + boolean appendMessageLogData(final long startOffset, final ByteBuffer data); + + boolean appendActionLogData(final long startOffset, final ByteBuffer data); + + void disableLagMonitor(String subject, String group); + + Table allPullLogs(); + + void destroyPullLog(final String subject, final String group, final String consumerId); +} diff --git a/qmq-store/src/main/java/qunar/tc/qmq/store/StorageConfig.java b/qmq-store/src/main/java/qunar/tc/qmq/store/StorageConfig.java new file mode 100644 index 00000000..a4896f07 --- /dev/null +++ b/qmq-store/src/main/java/qunar/tc/qmq/store/StorageConfig.java @@ -0,0 +1,53 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.store; + +/** + * @author keli.wang + * @since 2017/7/6 + */ +public interface StorageConfig { + String getCheckpointStorePath(); + + String getMessageLogStorePath(); + + long getMessageLogRetentionMs(); + + String getConsumerLogStorePath(); + + long getConsumerLogRetentionMs(); + + int getLogRetentionCheckIntervalSeconds(); + + String getPullLogStorePath(); + + long getPullLogRetentionMs(); + + String getActionLogStorePath(); + + boolean isDeleteExpiredLogsEnable(); + + long getLogRetentionMs(); + + int getRetryDelaySeconds(); + + int getCheckpointRetainCount(); + + long getActionCheckpointInterval(); + + long getMessageCheckpointInterval(); +} diff --git a/qmq-store/src/main/java/qunar/tc/qmq/store/StorageConfigImpl.java b/qmq-store/src/main/java/qunar/tc/qmq/store/StorageConfigImpl.java new file mode 100644 index 00000000..128bb6be --- /dev/null +++ b/qmq-store/src/main/java/qunar/tc/qmq/store/StorageConfigImpl.java @@ -0,0 +1,127 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.store; + +import qunar.tc.qmq.configuration.DynamicConfig; +import qunar.tc.qmq.constants.BrokerConstants; + +import java.io.File; +import java.util.concurrent.TimeUnit; + +/** + * @author keli.wang + * @since 2017/7/13 + */ +public class StorageConfigImpl implements StorageConfig { + private static final String CHECKPOINT = "checkpoint"; + private static final String MESSAGE_LOG = "messagelog"; + private static final String CONSUMER_LOG = "consumerlog"; + private static final String PULL_LOG = "pulllog"; + private static final String ACTION_LOG = "actionlog"; + + private static final long MS_PER_HOUR = TimeUnit.HOURS.toMillis(1); + + private final DynamicConfig config; + + public StorageConfigImpl(final DynamicConfig config) { + this.config = config; + } + + @Override + public String getCheckpointStorePath() { + return buildStorePath(CHECKPOINT); + } + + @Override + public String getMessageLogStorePath() { + return buildStorePath(MESSAGE_LOG); + } + + @Override + public long getMessageLogRetentionMs() { + final int retentionHours = config.getInt(BrokerConstants.MESSAGE_LOG_RETENTION_HOURS, BrokerConstants.DEFAULT_MESSAGE_LOG_RETENTION_HOURS); + return retentionHours * MS_PER_HOUR; + } + + @Override + public String getConsumerLogStorePath() { + return buildStorePath(CONSUMER_LOG); + } + + @Override + public long getConsumerLogRetentionMs() { + final int retentionHours = config.getInt(BrokerConstants.CONSUMER_LOG_RETENTION_HOURS, BrokerConstants.DEFAULT_CONSUMER_LOG_RETENTION_HOURS); + return retentionHours * MS_PER_HOUR; + } + + @Override + public int getLogRetentionCheckIntervalSeconds() { + return config.getInt(BrokerConstants.LOG_RETENTION_CHECK_INTERVAL_SECONDS, BrokerConstants.DEFAULT_LOG_RETENTION_CHECK_INTERVAL_SECONDS); + } + + @Override + public String getPullLogStorePath() { + return buildStorePath(PULL_LOG); + } + + @Override + public long getPullLogRetentionMs() { + final int retentionHours = config.getInt(BrokerConstants.PULL_LOG_RETENTION_HOURS, BrokerConstants.DEFAULT_PULL_LOG_RETENTION_HOURS); + return retentionHours * MS_PER_HOUR; + } + + @Override + public String getActionLogStorePath() { + return buildStorePath(ACTION_LOG); + } + + private String buildStorePath(final String name) { + final String root = config.getString(BrokerConstants.STORE_ROOT, BrokerConstants.LOG_STORE_ROOT); + return new File(root, name).getAbsolutePath(); + } + + @Override + public boolean isDeleteExpiredLogsEnable() { + return config.getBoolean(BrokerConstants.ENABLE_DELETE_EXPIRED_LOGS, false); + } + + @Override + public long getLogRetentionMs() { + final int retentionHours = config.getInt(BrokerConstants.PULL_LOG_RETENTION_HOURS, BrokerConstants.DEFAULT_PULL_LOG_RETENTION_HOURS); + return retentionHours * MS_PER_HOUR; + } + + @Override + public int getRetryDelaySeconds() { + return config.getInt(BrokerConstants.RETRY_DELAY_SECONDS, BrokerConstants.DEFAULT_RETRY_DELAY_SECONDS); + } + + @Override + public int getCheckpointRetainCount() { + return config.getInt(BrokerConstants.CHECKPOINT_RETAIN_COUNT, BrokerConstants.DEFAULT_CHECKPOINT_RETAIN_COUNT); + } + + @Override + public long getActionCheckpointInterval() { + return config.getLong(BrokerConstants.ACTION_CHECKPOINT_INTERVAL, BrokerConstants.DEFAULT_ACTION_CHECKPOINT_INTERVAL); + } + + @Override + public long getMessageCheckpointInterval() { + return config.getLong(BrokerConstants.MESSAGE_CHECKPOINT_INTERVAL, BrokerConstants.DEFAULT_MESSAGE_CHECKPOINT_INTERVAL); + } +} diff --git a/qmq-store/src/main/java/qunar/tc/qmq/store/StoreUtils.java b/qmq-store/src/main/java/qunar/tc/qmq/store/StoreUtils.java new file mode 100644 index 00000000..9b19710c --- /dev/null +++ b/qmq-store/src/main/java/qunar/tc/qmq/store/StoreUtils.java @@ -0,0 +1,41 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.store; + +import java.text.NumberFormat; + +/** + * @author keli.wang + * @since 2017/7/3 + */ +final class StoreUtils { + static String offset2FileName(final long offset) { + final NumberFormat nf = NumberFormat.getInstance(); + nf.setMinimumIntegerDigits(20); + nf.setMaximumFractionDigits(0); + nf.setGroupingUsed(false); + return nf.format(offset); + } + + static String offsetFileNameForSegment(final LogSegment segment) { + return offsetFileNameOf(segment.getBaseOffset()); + } + + static String offsetFileNameOf(final long baseOffset) { + return "." + StoreUtils.offset2FileName(baseOffset) + ".offset"; + } +} diff --git a/qmq-store/src/main/java/qunar/tc/qmq/store/action/ActionEvent.java b/qmq-store/src/main/java/qunar/tc/qmq/store/action/ActionEvent.java new file mode 100644 index 00000000..0ed1b332 --- /dev/null +++ b/qmq-store/src/main/java/qunar/tc/qmq/store/action/ActionEvent.java @@ -0,0 +1,41 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.store.action; + +import qunar.tc.qmq.store.Action; + +/** + * @author keli.wang + * @since 2017/10/16 + */ +public class ActionEvent { + private final long offset; + private final Action action; + + public ActionEvent(final long offset, final Action action) { + this.offset = offset; + this.action = action; + } + + public long getOffset() { + return offset; + } + + public Action getAction() { + return action; + } +} diff --git a/qmq-store/src/main/java/qunar/tc/qmq/store/action/ForeverOfflineAction.java b/qmq-store/src/main/java/qunar/tc/qmq/store/action/ForeverOfflineAction.java new file mode 100644 index 00000000..42145957 --- /dev/null +++ b/qmq-store/src/main/java/qunar/tc/qmq/store/action/ForeverOfflineAction.java @@ -0,0 +1,74 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.store.action; + +import qunar.tc.qmq.store.Action; +import qunar.tc.qmq.store.ActionType; + +/** + * @author keli.wang + * @since 2018/8/20 + */ +public class ForeverOfflineAction implements Action { + private final String subject; + private final String group; + private final String consumerId; + private final long timestamp; + + public ForeverOfflineAction(String subject, String group, String consumerId, long timestamp) { + this.subject = subject; + this.group = group; + this.consumerId = consumerId; + this.timestamp = timestamp; + } + + + @Override + public ActionType type() { + return ActionType.FOREVER_OFFLINE; + } + + @Override + public String subject() { + return subject; + } + + @Override + public String group() { + return group; + } + + @Override + public String consumerId() { + return consumerId; + } + + @Override + public long timestamp() { + return timestamp; + } + + @Override + public String toString() { + return "ForeverOfflineAction{" + + "subject='" + subject + '\'' + + ", group='" + group + '\'' + + ", consumerId='" + consumerId + '\'' + + ", timestamp=" + timestamp + + '}'; + } +} diff --git a/qmq-store/src/main/java/qunar/tc/qmq/store/action/ForeverOfflineActionReaderWriter.java b/qmq-store/src/main/java/qunar/tc/qmq/store/action/ForeverOfflineActionReaderWriter.java new file mode 100644 index 00000000..a0db8a1b --- /dev/null +++ b/qmq-store/src/main/java/qunar/tc/qmq/store/action/ForeverOfflineActionReaderWriter.java @@ -0,0 +1,51 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.store.action; + +import qunar.tc.qmq.store.Action; +import qunar.tc.qmq.store.ActionReaderWriter; +import qunar.tc.qmq.utils.PayloadHolderUtils; + +import java.nio.ByteBuffer; + +/** + * @author keli.wang + * @since 2018/8/20 + */ +public class ForeverOfflineActionReaderWriter implements ActionReaderWriter { + @Override + public int write(ByteBuffer to, Action action) { + final int startIndex = to.position(); + + PayloadHolderUtils.writeString(action.subject(), to); + PayloadHolderUtils.writeString(action.group(), to); + PayloadHolderUtils.writeString(action.consumerId(), to); + to.putLong(action.timestamp()); + + return to.position() - startIndex; + } + + @Override + public Action read(ByteBuffer from) { + final String subject = PayloadHolderUtils.readString(from); + final String group = PayloadHolderUtils.readString(from); + final String consumerId = PayloadHolderUtils.readString(from); + + final long timestamp = from.getLong(); + return new ForeverOfflineAction(subject, group, consumerId, timestamp); + } +} diff --git a/qmq-store/src/main/java/qunar/tc/qmq/store/action/MaxSequencesUpdater.java b/qmq-store/src/main/java/qunar/tc/qmq/store/action/MaxSequencesUpdater.java new file mode 100644 index 00000000..9b3eafc4 --- /dev/null +++ b/qmq-store/src/main/java/qunar/tc/qmq/store/action/MaxSequencesUpdater.java @@ -0,0 +1,47 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.store.action; + +import qunar.tc.qmq.store.CheckpointManager; +import qunar.tc.qmq.store.event.FixedExecOrderEventBus; + +/** + * @author keli.wang + * @since 2017/8/21 + */ +public class MaxSequencesUpdater implements FixedExecOrderEventBus.Listener { + private final CheckpointManager manager; + + public MaxSequencesUpdater(final CheckpointManager manager) { + this.manager = manager; + } + + @Override + public void onEvent(final ActionEvent event) { + final long offset = event.getOffset(); + + switch (event.getAction().type()) { + case PULL: + manager.updateActionReplayState(offset, (PullAction) event.getAction()); + break; + case RANGE_ACK: + manager.updateActionReplayState(offset, (RangeAckAction) event.getAction()); + break; + } + + } +} diff --git a/qmq-store/src/main/java/qunar/tc/qmq/store/action/PullAction.java b/qmq-store/src/main/java/qunar/tc/qmq/store/action/PullAction.java new file mode 100644 index 00000000..43d779cb --- /dev/null +++ b/qmq-store/src/main/java/qunar/tc/qmq/store/action/PullAction.java @@ -0,0 +1,134 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.store.action; + +import com.google.common.base.Preconditions; +import qunar.tc.qmq.store.Action; +import qunar.tc.qmq.store.ActionType; + +/** + * @author keli.wang + * @since 2017/8/20 + */ +public class PullAction implements Action { + private final String subject; + private final String group; + private final String consumerId; + private final long timestamp; + private final boolean broadcast; + + //first sequence of pull log + private final long firstSequence; + + //last sequence of pull log + private final long lastSequence; + + //fist sequence of consumer log + private final long firstMessageSequence; + + //last sequence of consumer log + private final long lastMessageSequence; + + public PullAction(final String subject, final String group, final String consumerId, long timestamp, boolean broadcast, + long firstSequence, long lastSequence, + long firstMessageSequence, long lastMessageSequence) { + Preconditions.checkArgument(lastSequence - firstSequence == lastMessageSequence - firstMessageSequence); + + this.subject = subject; + this.group = group; + this.consumerId = consumerId; + this.timestamp = timestamp; + this.broadcast = broadcast; + + this.firstSequence = firstSequence; + this.lastSequence = lastSequence; + + this.firstMessageSequence = firstMessageSequence; + this.lastMessageSequence = lastMessageSequence; + } + + @Override + public ActionType type() { + return ActionType.PULL; + } + + @Override + public String subject() { + return subject; + } + + @Override + public String group() { + return group; + } + + @Override + public String consumerId() { + return consumerId; + } + + @Override + public long timestamp() { + return timestamp; + } + + public boolean isBroadcast() { + return broadcast; + } + + /** + * 在pull log中第一个偏移 + */ + public long getFirstSequence() { + return firstSequence; + } + + /** + * 在pull log中最后一个偏移 + */ + public long getLastSequence() { + return lastSequence; + } + + /** + * 在consuemr log中第一个偏移 + */ + public long getFirstMessageSequence() { + return firstMessageSequence; + } + + /** + * 在consumer log中最后一个偏移 + */ + public long getLastMessageSequence() { + return lastMessageSequence; + } + + @Override + public String toString() { + return "PullAction{" + + "subject='" + subject + '\'' + + ", group='" + group + '\'' + + ", consumerId='" + consumerId + '\'' + + ", broadcast=" + broadcast + + ", firstSequence=" + firstSequence + + ", lastSequence=" + lastSequence + + ", firstMessageSequence=" + firstMessageSequence + + ", lastMessageSequence=" + lastMessageSequence + + '}'; + } +} diff --git a/qmq-store/src/main/java/qunar/tc/qmq/store/action/PullActionReaderWriter.java b/qmq-store/src/main/java/qunar/tc/qmq/store/action/PullActionReaderWriter.java new file mode 100644 index 00000000..33887402 --- /dev/null +++ b/qmq-store/src/main/java/qunar/tc/qmq/store/action/PullActionReaderWriter.java @@ -0,0 +1,80 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.store.action; + +import qunar.tc.qmq.store.Action; +import qunar.tc.qmq.store.ActionReaderWriter; +import qunar.tc.qmq.utils.PayloadHolderUtils; + +import java.nio.ByteBuffer; +import java.util.Objects; + +/** + * @author keli.wang + * @since 2017/8/20 + */ +public class PullActionReaderWriter implements ActionReaderWriter { + private static final byte TRUE_BYTE = (byte) 1; + private static final byte FALSE_BYTE = (byte) 0; + + @Override + public int write(final ByteBuffer to, final Action action) { + final int startIndex = to.position(); + + final PullAction pull = (PullAction) action; + PayloadHolderUtils.writeString(pull.subject(), to); + PayloadHolderUtils.writeString(pull.group(), to); + PayloadHolderUtils.writeString(pull.consumerId(), to); + + to.putLong(action.timestamp()); + to.put(toByte(pull.isBroadcast())); + + to.putLong(pull.getFirstSequence()); + to.putLong(pull.getLastSequence()); + + to.putLong(pull.getFirstMessageSequence()); + to.putLong(pull.getLastMessageSequence()); + + return to.position() - startIndex; + } + + @Override + public PullAction read(final ByteBuffer from) { + final String subject = PayloadHolderUtils.readString(from); + final String group = PayloadHolderUtils.readString(from); + final String consumerId = PayloadHolderUtils.readString(from); + + final long timestamp = from.getLong(); + final boolean broadcast = fromByte(from.get()); + + final long firstSequence = from.getLong(); + final long lastSequence = from.getLong(); + + final long firstMessageSequence = from.getLong(); + final long lastMessageSequence = from.getLong(); + + return new PullAction(subject, group, consumerId, timestamp, broadcast, firstSequence, lastSequence, firstMessageSequence, lastMessageSequence); + } + + private byte toByte(final boolean bool) { + return bool ? TRUE_BYTE : FALSE_BYTE; + } + + private boolean fromByte(final byte b) { + return Objects.equals(b, TRUE_BYTE); + } +} diff --git a/qmq-store/src/main/java/qunar/tc/qmq/store/action/PullLogBuilder.java b/qmq-store/src/main/java/qunar/tc/qmq/store/action/PullLogBuilder.java new file mode 100644 index 00000000..fa0870f9 --- /dev/null +++ b/qmq-store/src/main/java/qunar/tc/qmq/store/action/PullLogBuilder.java @@ -0,0 +1,63 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.store.action; + +import qunar.tc.qmq.store.Storage; +import qunar.tc.qmq.store.PullLogMessage; +import qunar.tc.qmq.store.event.FixedExecOrderEventBus; + +import java.util.ArrayList; +import java.util.List; + +/** + * @author keli.wang + * @since 2017/8/21 + */ +public class PullLogBuilder implements FixedExecOrderEventBus.Listener { + private final Storage storage; + + public PullLogBuilder(final Storage storage) { + this.storage = storage; + } + + @Override + public void onEvent(final ActionEvent event) { + switch (event.getAction().type()) { + case PULL: + buildPullLog(event); + break; + } + } + + private void buildPullLog(ActionEvent event) { + final PullAction action = (PullAction) event.getAction(); + if (action.isBroadcast()) return; + + if (action.getFirstSequence() - action.getLastSequence() > 0) return; + storage.putPullLogs(action.subject(), action.group(), action.consumerId(), createMessages(action)); + } + + private List createMessages(final PullAction action) { + final int count = (int) (action.getLastSequence() - action.getFirstSequence() + 1); + final List messages = new ArrayList<>(count); + for (int i = 0; i < count; i++) { + messages.add(new PullLogMessage(action.getFirstSequence() + i, action.getFirstMessageSequence() + i)); + } + + return messages; + } +} diff --git a/qmq-store/src/main/java/qunar/tc/qmq/store/action/RangeAckAction.java b/qmq-store/src/main/java/qunar/tc/qmq/store/action/RangeAckAction.java new file mode 100644 index 00000000..9ba7d342 --- /dev/null +++ b/qmq-store/src/main/java/qunar/tc/qmq/store/action/RangeAckAction.java @@ -0,0 +1,88 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.store.action; + +import qunar.tc.qmq.store.Action; +import qunar.tc.qmq.store.ActionType; + +/** + * @author yunfeng.yang + * @since 2017/8/28 + */ +public class RangeAckAction implements Action { + private final String subject; + private final String group; + private final String consumerId; + private final long timestamp; + + private final long firstSequence; + private final long lastSequence; + + public RangeAckAction(String subject, String group, String consumerId, long timestamp, long firstSequence, long lastSequence) { + this.subject = subject; + this.group = group; + this.consumerId = consumerId; + this.timestamp = timestamp; + + this.firstSequence = firstSequence; + this.lastSequence = lastSequence; + } + + @Override + public ActionType type() { + return ActionType.RANGE_ACK; + } + + @Override + public String subject() { + return subject; + } + + @Override + public String group() { + return group; + } + + @Override + public String consumerId() { + return consumerId; + } + + @Override + public long timestamp() { + return timestamp; + } + + public long getFirstSequence() { + return firstSequence; + } + + public long getLastSequence() { + return lastSequence; + } + + @Override + public String toString() { + return "RangeAckAction{" + + "subject='" + subject + '\'' + + ", group='" + group + '\'' + + ", consumerId='" + consumerId + '\'' + + ", firstSequence=" + firstSequence + + ", lastSequence=" + lastSequence + + '}'; + } +} diff --git a/qmq-store/src/main/java/qunar/tc/qmq/store/action/RangeAckActionReaderWriter.java b/qmq-store/src/main/java/qunar/tc/qmq/store/action/RangeAckActionReaderWriter.java new file mode 100644 index 00000000..dee7a10a --- /dev/null +++ b/qmq-store/src/main/java/qunar/tc/qmq/store/action/RangeAckActionReaderWriter.java @@ -0,0 +1,55 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.store.action; + +import qunar.tc.qmq.store.Action; +import qunar.tc.qmq.store.ActionReaderWriter; +import qunar.tc.qmq.utils.PayloadHolderUtils; + +import java.nio.ByteBuffer; + +/** + * @author keli.wang + * @since 2017/8/20 + */ +public class RangeAckActionReaderWriter implements ActionReaderWriter { + @Override + public int write(ByteBuffer to, Action action) { + final int startIndex = to.position(); + + final RangeAckAction rangeAck = (RangeAckAction) action; + PayloadHolderUtils.writeString(rangeAck.subject(), to); + PayloadHolderUtils.writeString(rangeAck.group(), to); + PayloadHolderUtils.writeString(rangeAck.consumerId(), to); + to.putLong(action.timestamp()); + to.putLong(rangeAck.getFirstSequence()); + to.putLong(rangeAck.getLastSequence()); + return to.position() - startIndex; + } + + @Override + public RangeAckAction read(final ByteBuffer from) { + final String subject = PayloadHolderUtils.readString(from); + final String group = PayloadHolderUtils.readString(from); + final String consumerId = PayloadHolderUtils.readString(from); + final long timestamp = from.getLong(); + final long firstSequence = from.getLong(); + final long lastSequence = from.getLong(); + + return new RangeAckAction(subject, group, consumerId, timestamp, firstSequence, lastSequence); + } +} diff --git a/qmq-store/src/main/java/qunar/tc/qmq/store/event/FixedExecOrderEventBus.java b/qmq-store/src/main/java/qunar/tc/qmq/store/event/FixedExecOrderEventBus.java new file mode 100644 index 00000000..883aec2c --- /dev/null +++ b/qmq-store/src/main/java/qunar/tc/qmq/store/event/FixedExecOrderEventBus.java @@ -0,0 +1,75 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.store.event; + +import com.google.common.collect.ArrayListMultimap; +import com.google.common.collect.ListMultimap; + +import java.util.List; +import java.util.concurrent.locks.Lock; +import java.util.concurrent.locks.ReadWriteLock; +import java.util.concurrent.locks.ReentrantReadWriteLock; + +/** + * An event bus that execute listenersByType in fixed order + * + * @author keli.wang + * @since 2018/7/13 + */ +public class FixedExecOrderEventBus { + private final ReadWriteLock guard; + private final ListMultimap, Listener> listenersByType; + + public FixedExecOrderEventBus() { + this.guard = new ReentrantReadWriteLock(); + this.listenersByType = ArrayListMultimap.create(); + } + + public void subscribe(final Class clazz, final Listener listener) { + final Lock lock = guard.writeLock(); + lock.lock(); + try { + listenersByType.put(clazz, listener); + } finally { + lock.unlock(); + } + } + + @SuppressWarnings("unchecked") + public void post(final Object event) { + final Lock lock = guard.readLock(); + + lock.lock(); + + try { + final List listeners = listenersByType.get(event.getClass()); + if (listeners == null) { + throw new RuntimeException("unsupported event type " + event.getClass().getSimpleName()); + } + + for (final Listener listener : listeners) { + listener.onEvent(event); + } + } finally { + lock.unlock(); + } + } + + public interface Listener { + void onEvent(final E event); + } +} diff --git a/qmq-store/src/main/resources/META-INF/native/libmmaputil.so b/qmq-store/src/main/resources/META-INF/native/libmmaputil.so new file mode 100644 index 0000000000000000000000000000000000000000..e0c8599b0ab43e4541485d0a265a06b8036e3e4b GIT binary patch literal 6146 zcmcIoYit}>6+Y{=W1A%QHl);Tf-;I4$su&(yoyr@*s(X61Un9nje-K2jCaTJM*DDf zXTzo<6v-)cag9u&RfUA`4?w8>f%t=*0+L&|L_raC{HT>L6_bc6o zyP~Sz#zqeUB{CEsG1l;Zo0hjVdN^pLPN|WhDeQz5J5$@4+75I^zY1uR=Z55U(w~)}HRwhUCQ#UYhV8V8)_8B?&r^Jb`Ty{-VcHPhw)6J4b`IQ{ z|Lo%Y*pFX%_MPHo@g#Ebk2*dvXM~LUu4ci+mwc`lUVrLKJ+a@uwRrC2=F3xeTK;nV z?Upb8{trK6(R;QqHPesqB3e`87&WYRtb%W5d}|H;0%IF$;4_Rx=uWw_*{G0CX}mpv ze|r@?!uW<7`F}{^>*$HVM>(eOwe)j6h`+(SYL;~-oy}Nz*Uq_?MOHGCbjccJLY5UT z+S0XC$wwWsj-Iy$oioY2>*RV;c0TXqDU}?_yNURY9npL?`Y;`}&)e3yLdMQnZrnPT zKF5BuImhZx+vCG-GG&eC9H)cEv-xDv8nqMWlX*vc#65G%N!YHibpgR5w`TjhCAQRk`dPyrhkZdN_{He!MSEHeN7?AaR0huPQ2B$^eI+edQ~1z^LR8TUT&H% z?>ks;PeA9J$duVI7C$LHL-@`7?RjW+z1 z0}(bLvvjk2=tN<}oH($+y5z1irw+8%8MO2r@iO({zlRL!o7#4R3H!UrI(^$;H(&f$ z*qnGXWbR+g8?9SsqW8TogZ!8a*3pI zyqS8~d1iz4JmM$)urNTmK_?lVjpX)2d zX3fcA{{O<HcMB^*SUb^Dal6~a*`s?@`_@#ke!>CbJnzd(*;ja=C1d zLYu-LmvvG;Y(0OG4yMwu4u>8cq);&u+O)2*N!I!YSte`JH4dTXcy~+Wt93o=*PNaR z*Ckm=jD3w=ss<>?1IN6OJfz7Gl?Gnsn6ws;&<`|SrzzIi^_s`}^v=6wsWHuJM-5uu zrY9Gcv1KjC_WY98TMK$H**uee@o)0E4s z@?X}0xqgaAU2bdsRW1K#K(5{V-wA#<1Oxp}zvnGY5BKyuVzl#)yxrIv-5dRwvAbi} z?v6dX_89GhPQoy4*TWy$vz;iKzmRtA5vFd=qhsi0vaS<7lPN?;3dvOBp=5#-H)iL@ zD4MvCVM~v?IZtxl$>nAED>cnnEXz45TMTq>JmpeU?NL$JDRN)!RMA{k?Ho~OjJFs& z?O0<8wgpGgIB!FF4(O4yaTbb!cxPLciJeZyIeM0T@*}DO@jk@1*>u{;xWUT(Z~P`U zlQD*Ix!UhDc-Nl=EJeBZ4<19X$Gidkgx177!1#rOPVUH@fj#CGsG$e0#1$RbW4w1U zCca^hc?$Xx%lr-V0GP=gnIo{ryZ}WUe~FF!HB1Ta;N1kn zJO@23AUDK^J@U%1%x{l%0~G6w*vncKv@bFyd0}0{x(JFmtfHF4^0z!Qe)~DC0E+mq zfqg^+Uk=#g`~&)I;Df;q{w@XVv5tdc{f9o{ga5j=$M<_i7XoxE6g3T`Lr(?lgXb0O zi*OT+KNGND(D6ZgVW=s-AK>S-J;n#laX8-v>lL*BNx;6n(NM12^~Uqx>}Lb^o!Y)r z+Xv$X`CkO=v9EW%Py1i7y_5{^k9`H_)EWK3Pw3N~xvS~*K5$4LVh%3xWrZCnL!XM6)y&zC6U!QTSn^Qh7fdM9!eUVXpyB;#`5hVKYV z`JGei9`3uV=zm}NudzPF?NrKlg+}q5n|BMNqtQ6uWAzLT3|f6hP7KM-n$=su9qsXn z5?Sj^Dm!AQEOkV&>_UaLc52+})nv;AF_YEBE?z09C9Xb&kvWB`3_QfRcL_TYc*_lL&?*P4* u&<=Y)cdWbrNDrfxcZfAE3Txdcc6Ish@hUC_D@E_~TCwWh5ci4dy!amp&hX;^ literal 0 HcmV?d00001 diff --git a/qmq-sync/pom.xml b/qmq-sync/pom.xml new file mode 100644 index 00000000..25d8a91e --- /dev/null +++ b/qmq-sync/pom.xml @@ -0,0 +1,38 @@ + + + 4.0.0 + + + qmq + qunar.tc + 4.0.30 + + + qmq-sync + + + + qunar.tc + qmq-remoting + + + qunar.tc + qmq-server-common + + + qunar.tc + qmq-store + + + + org.slf4j + slf4j-api + + + qunar.tc + qmq-common + + + \ No newline at end of file diff --git a/qmq-sync/src/main/java/qunar/tc/qmq/sync/AbstractSyncLogProcessor.java b/qmq-sync/src/main/java/qunar/tc/qmq/sync/AbstractSyncLogProcessor.java new file mode 100644 index 00000000..259e7aad --- /dev/null +++ b/qmq-sync/src/main/java/qunar/tc/qmq/sync/AbstractSyncLogProcessor.java @@ -0,0 +1,44 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.sync; + +import io.netty.buffer.ByteBuf; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import qunar.tc.qmq.protocol.Datagram; + +/** + * @author yunfeng.yang + * @since 2017/8/19 + */ +public abstract class AbstractSyncLogProcessor implements SyncLogProcessor { + private static final Logger LOG = LoggerFactory.getLogger(AbstractSyncLogProcessor.class); + + @Override + public void process(Datagram datagram) { + final ByteBuf body = datagram.getBody(); + final int size = body.readInt(); + if (size == 0) { + LOG.debug("sync data empty"); + return; + } + final long startOffset = body.readLong(); + appendLogs(startOffset, body); + } + + public abstract void appendLogs(long startOffset, ByteBuf body); +} diff --git a/qmq-sync/src/main/java/qunar/tc/qmq/sync/DelaySyncRequest.java b/qmq-sync/src/main/java/qunar/tc/qmq/sync/DelaySyncRequest.java new file mode 100644 index 00000000..ae332f77 --- /dev/null +++ b/qmq-sync/src/main/java/qunar/tc/qmq/sync/DelaySyncRequest.java @@ -0,0 +1,112 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.sync; + +/** + * @author xufeng.deng dennisdxf@gmail.com + * @since 2018-08-10 11:44 + */ +public class DelaySyncRequest { + private final long messageLogOffset; + private final DispatchLogSyncRequest dispatchSyncRequest; + private final int syncType; + + public DelaySyncRequest(long messageLogOffset, DispatchLogSyncRequest dispatchLogSyncRequest, int syncType) { + this.messageLogOffset = messageLogOffset; + this.dispatchSyncRequest = dispatchLogSyncRequest; + this.syncType = syncType; + } + + public DelaySyncRequest(long messageLogOffset, int dispatchLogSegmentBaseOffset, long dispatchLogOffset, int lastDispatchLogBaseOffset, long lastDispatchLogOffset, int syncType) { + this.messageLogOffset = messageLogOffset; + this.dispatchSyncRequest = new DispatchLogSyncRequest(dispatchLogSegmentBaseOffset, dispatchLogOffset, lastDispatchLogBaseOffset, lastDispatchLogOffset); + this.syncType = syncType; + } + + public long getMessageLogOffset() { + return messageLogOffset; + } + + public int getDispatchSegmentBaseOffset() { + return dispatchSyncRequest == null ? -1 : dispatchSyncRequest.getSegmentBaseOffset(); + } + + public long getDispatchLogOffset() { + return dispatchSyncRequest == null ? -1 : dispatchSyncRequest.getDispatchLogOffset(); + } + + public int getLastDispatchSegmentBaseOffset() { + return dispatchSyncRequest == null ? -1 : dispatchSyncRequest.getLastSegmentBaseOffset(); + } + + public long getLastDispatchSegmentOffset() { + return dispatchSyncRequest == null ? -1 : dispatchSyncRequest.getLastDispatchLogOffset(); + } + + public int getSyncType() { + return syncType; + } + + @Override + public String toString() { + return "DelaySyncRequest{" + + "messageLogOffset=" + messageLogOffset + + ", dispatchSyncRequest=" + dispatchSyncRequest + + ", syncType=" + syncType + + '}'; + } + + public static class DispatchLogSyncRequest { + private final int segmentBaseOffset; + private final long dispatchLogOffset; + private final int lastSegmentBaseOffset; + private final long lastDispatchLogOffset; + + public DispatchLogSyncRequest(int segmentBaseOffset, long dispatchLogOffset, int lastSegmentBaseOffset, long lastDispatchLogOffset) { + this.segmentBaseOffset = segmentBaseOffset; + this.dispatchLogOffset = dispatchLogOffset; + this.lastSegmentBaseOffset = lastSegmentBaseOffset; + this.lastDispatchLogOffset = lastDispatchLogOffset; + } + + public int getSegmentBaseOffset() { + return segmentBaseOffset; + } + + public long getDispatchLogOffset() { + return dispatchLogOffset; + } + + public int getLastSegmentBaseOffset() { + return lastSegmentBaseOffset; + } + + public long getLastDispatchLogOffset() { + return lastDispatchLogOffset; + } + + @Override + public String toString() { + return "DispatchLogSyncRequest{" + + "segmentBaseOffset=" + segmentBaseOffset + + ", dispatchLogOffset=" + dispatchLogOffset + + ", lastSegmentBaseOffset=" + lastSegmentBaseOffset + + ", lastDispatchLogOffset=" + lastDispatchLogOffset + + '}'; + } + } +} \ No newline at end of file diff --git a/qmq-sync/src/main/java/qunar/tc/qmq/sync/HeartbeatProcessor.java b/qmq-sync/src/main/java/qunar/tc/qmq/sync/HeartbeatProcessor.java new file mode 100644 index 00000000..e94764cd --- /dev/null +++ b/qmq-sync/src/main/java/qunar/tc/qmq/sync/HeartbeatProcessor.java @@ -0,0 +1,59 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.sync; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import qunar.tc.qmq.base.SyncRequest; +import qunar.tc.qmq.protocol.Datagram; +import qunar.tc.qmq.store.Storage; + +import java.util.concurrent.TimeUnit; + +import static qunar.tc.qmq.constants.BrokerConstants.DEFAULT_HEARTBEAT_SLEEP_TIMEOUT_MS; + +/** + * @author yunfeng.yang + * @since 2017/8/19 + */ +public class HeartbeatProcessor implements SyncLogProcessor { + private static final Logger LOG = LoggerFactory.getLogger(HeartbeatProcessor.class); + + private final Storage storage; + private final long sleepTimeoutMs; + + public HeartbeatProcessor(Storage storage) { + this.storage = storage; + this.sleepTimeoutMs = DEFAULT_HEARTBEAT_SLEEP_TIMEOUT_MS; + } + + @Override + public void process(Datagram datagram) { + try { + TimeUnit.MILLISECONDS.sleep(sleepTimeoutMs); + } catch (InterruptedException e) { + LOG.error("heart beat sleep error", e); + } + } + + @Override + public SyncRequest getRequest() { + long messageLogMaxOffset = storage.getMaxMessageOffset(); + long actionLogMaxOffset = storage.getMaxActionLogOffset(); + return new SyncRequest(SyncType.heartbeat.getCode(), messageLogMaxOffset, actionLogMaxOffset); + } +} diff --git a/qmq-sync/src/main/java/qunar/tc/qmq/sync/MasterSlaveSyncManager.java b/qmq-sync/src/main/java/qunar/tc/qmq/sync/MasterSlaveSyncManager.java new file mode 100644 index 00000000..d8bdae64 --- /dev/null +++ b/qmq-sync/src/main/java/qunar/tc/qmq/sync/MasterSlaveSyncManager.java @@ -0,0 +1,118 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.sync; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import qunar.tc.qmq.common.Disposable; +import qunar.tc.qmq.configuration.BrokerConfig; +import qunar.tc.qmq.monitor.QMon; +import qunar.tc.qmq.protocol.Datagram; + +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.TimeUnit; + +/** + * @author yunfeng.yang + * @since 2017/8/18 + */ +public class MasterSlaveSyncManager implements Disposable { + private static final Logger LOG = LoggerFactory.getLogger(MasterSlaveSyncManager.class); + + private final SlaveSyncClient slaveSyncClient; + private final Map slaveSyncTasks; + + public MasterSlaveSyncManager(final SlaveSyncClient slaveSyncClient) { + this.slaveSyncClient = slaveSyncClient; + this.slaveSyncTasks = new HashMap<>(); + } + + public void registerProcessor(final SyncType syncType, final SyncLogProcessor syncLogProcessor) { + slaveSyncTasks.put(syncType, new SlaveSyncTask(syncLogProcessor)); + } + + public void startSync() { + for (final Map.Entry entry : slaveSyncTasks.entrySet()) { + new Thread(entry.getValue(), "sync-task-" + entry.getKey().name()).start(); + } + } + + @Override + public void destroy() { + for (SlaveSyncTask task : slaveSyncTasks.values()) { + try { + task.shutdown(); + } catch (Exception e) { + LOG.error("disposable destroy failed", e); + } + } + } + + private class SlaveSyncTask implements Runnable { + private final SyncLogProcessor processor; + private final String processorName; + + private volatile boolean running = true; + + SlaveSyncTask(SyncLogProcessor processor) { + this.processor = processor; + this.processorName = processor.getClass().getSimpleName(); + } + + @Override + public void run() { + while (running) { + final long start = System.currentTimeMillis(); + try { + doSync(); + } catch (Throwable e) { + QMon.syncTaskSyncFailedCountInc(BrokerConfig.getBrokerName()); + LOG.error("[{}] sync data from master failed, will retry after 2 seconds.", processorName, e); + + silentSleep(2); + } finally { + QMon.syncTaskExecTimer(processorName, System.currentTimeMillis() - start); + } + } + } + + private void doSync() { + Datagram response = null; + try { + response = slaveSyncClient.syncLog(processor.getRequest()); + processor.process(response); + } finally { + if (response != null) { + response.release(); + } + } + } + + private void silentSleep(final long timeoutInSeconds) { + try { + TimeUnit.SECONDS.sleep(timeoutInSeconds); + } catch (InterruptedException ignore) { + LOG.debug("sleep interrupted"); + } + } + + void shutdown() { + running = false; + } + } +} diff --git a/qmq-sync/src/main/java/qunar/tc/qmq/sync/SlaveSyncClient.java b/qmq-sync/src/main/java/qunar/tc/qmq/sync/SlaveSyncClient.java new file mode 100644 index 00000000..982d5731 --- /dev/null +++ b/qmq-sync/src/main/java/qunar/tc/qmq/sync/SlaveSyncClient.java @@ -0,0 +1,84 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.sync; + +import io.netty.buffer.ByteBuf; +import qunar.tc.qmq.base.SyncRequest; +import qunar.tc.qmq.configuration.BrokerConfig; +import qunar.tc.qmq.configuration.DynamicConfig; +import qunar.tc.qmq.netty.NettyClientConfig; +import qunar.tc.qmq.netty.client.NettyClient; +import qunar.tc.qmq.protocol.CommandCode; +import qunar.tc.qmq.protocol.Datagram; +import qunar.tc.qmq.protocol.PayloadHolder; +import qunar.tc.qmq.util.RemotingBuilder; + +/** + * @author keli.wang + * @since 2018/10/29 + */ +public class SlaveSyncClient { + private final NettyClient client; + private final String master; + private volatile long timeout; + + public SlaveSyncClient(DynamicConfig config) { + this.client = NettyClient.getClient(); + this.client.start(new NettyClientConfig()); + this.master = BrokerConfig.getMasterAddress(); + + config.addListener(conf -> timeout = conf.getLong("slave.sync.timeout", 3000L)); + } + + public Datagram syncCheckpoint() { + final Datagram datagram = RemotingBuilder.buildRequestDatagram(CommandCode.SYNC_CHECKPOINT_REQUEST, null); + try { + return client.sendSync(master, datagram, timeout); + } catch (Throwable e) { + throw new RuntimeException(String.format("sync checkpoint failed. master: %s, timeout: %d", master, timeout), e); + } + } + + public Datagram syncLog(final SyncRequest request) { + final Datagram datagram = newSyncLogRequest(request); + try { + return client.sendSync(master, datagram, timeout); + } catch (Throwable e) { + throw new RuntimeException(String.format("sync log failed. master: %s, timeout: %d", master, timeout), e); + } + } + + private Datagram newSyncLogRequest(final SyncRequest request) { + final SyncRequestPayloadHolder payloadHolder = new SyncRequestPayloadHolder(request); + return RemotingBuilder.buildRequestDatagram(CommandCode.SYNC_LOG_REQUEST, payloadHolder); + } + + private class SyncRequestPayloadHolder implements PayloadHolder { + private final SyncRequest request; + + SyncRequestPayloadHolder(SyncRequest request) { + this.request = request; + } + + @Override + public void writeBody(ByteBuf out) { + out.writeByte(request.getSyncType()); + out.writeLong(request.getMessageLogOffset()); + out.writeLong(request.getActionLogOffset()); + } + } +} diff --git a/qmq-sync/src/main/java/qunar/tc/qmq/sync/SlaveSyncSender.java b/qmq-sync/src/main/java/qunar/tc/qmq/sync/SlaveSyncSender.java new file mode 100644 index 00000000..6ba07246 --- /dev/null +++ b/qmq-sync/src/main/java/qunar/tc/qmq/sync/SlaveSyncSender.java @@ -0,0 +1,41 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.sync; + +import qunar.tc.qmq.configuration.DynamicConfig; +import qunar.tc.qmq.netty.client.NettyClient; +import qunar.tc.qmq.netty.exception.ClientSendException; +import qunar.tc.qmq.netty.exception.RemoteTimeoutException; +import qunar.tc.qmq.protocol.Datagram; + +/** + * @author yunfeng.yang + * @since 2017/8/18 + */ +public class SlaveSyncSender { + private final NettyClient client; + private final long timeout; + + public SlaveSyncSender(DynamicConfig config, NettyClient client) { + this.timeout = config.getLong("slave.sync.timeout", 3000L); + this.client = client; + } + + public Datagram send(String address, Datagram datagram) throws InterruptedException, RemoteTimeoutException, ClientSendException { + return client.sendSync(address, datagram, timeout); + } +} diff --git a/qmq-sync/src/main/java/qunar/tc/qmq/sync/SyncActionLogProcessor.java b/qmq-sync/src/main/java/qunar/tc/qmq/sync/SyncActionLogProcessor.java new file mode 100644 index 00000000..9f7fba6d --- /dev/null +++ b/qmq-sync/src/main/java/qunar/tc/qmq/sync/SyncActionLogProcessor.java @@ -0,0 +1,45 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.sync; + +import io.netty.buffer.ByteBuf; +import qunar.tc.qmq.base.SyncRequest; +import qunar.tc.qmq.store.Storage; + +/** + * @author yunfeng.yang + * @since 2017/8/19 + */ +public class SyncActionLogProcessor extends AbstractSyncLogProcessor { + + private final Storage storage; + + public SyncActionLogProcessor(Storage storage) { + this.storage = storage; + } + + @Override + public void appendLogs(long startOffset, ByteBuf body) { + storage.appendActionLogData(startOffset, body.nioBuffer()); + } + + @Override + public SyncRequest getRequest() { + final long actionLogMaxOffset = storage.getMaxActionLogOffset(); + return new SyncRequest(SyncType.action.getCode(), 0, actionLogMaxOffset); + } +} \ No newline at end of file diff --git a/qmq-sync/src/main/java/qunar/tc/qmq/sync/SyncLogProcessor.java b/qmq-sync/src/main/java/qunar/tc/qmq/sync/SyncLogProcessor.java new file mode 100644 index 00000000..df37f5e9 --- /dev/null +++ b/qmq-sync/src/main/java/qunar/tc/qmq/sync/SyncLogProcessor.java @@ -0,0 +1,30 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.sync; + +import qunar.tc.qmq.base.SyncRequest; +import qunar.tc.qmq.protocol.Datagram; + +/** + * @author yunfeng.yang + * @since 2017/8/18 + */ +public interface SyncLogProcessor { + void process(Datagram syncData); + + SyncRequest getRequest(); +} diff --git a/qmq-sync/src/main/java/qunar/tc/qmq/sync/SyncMessageLogProcessor.java b/qmq-sync/src/main/java/qunar/tc/qmq/sync/SyncMessageLogProcessor.java new file mode 100644 index 00000000..c2b4d053 --- /dev/null +++ b/qmq-sync/src/main/java/qunar/tc/qmq/sync/SyncMessageLogProcessor.java @@ -0,0 +1,45 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.sync; + +import io.netty.buffer.ByteBuf; +import qunar.tc.qmq.base.SyncRequest; +import qunar.tc.qmq.store.Storage; + +/** + * @author yunfeng.yang + * @since 2017/8/19 + */ +public class SyncMessageLogProcessor extends AbstractSyncLogProcessor { + + private final Storage storage; + + public SyncMessageLogProcessor(Storage storage) { + this.storage = storage; + } + + @Override + public void appendLogs(long startOffset, ByteBuf body) { + storage.appendMessageLogData(startOffset, body.nioBuffer()); + } + + @Override + public SyncRequest getRequest() { + final long messageLogMaxOffset = storage.getMaxMessageOffset(); + return new SyncRequest(SyncType.message.getCode(), messageLogMaxOffset, 0L); + } +} diff --git a/qmq-sync/src/main/java/qunar/tc/qmq/sync/SyncType.java b/qmq-sync/src/main/java/qunar/tc/qmq/sync/SyncType.java new file mode 100644 index 00000000..8d8344a0 --- /dev/null +++ b/qmq-sync/src/main/java/qunar/tc/qmq/sync/SyncType.java @@ -0,0 +1,35 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.sync; + +/** + * @author yunfeng.yang + * @since 2017/8/18 + */ +public enum SyncType { + action(1), message(2), heartbeat(3), dispatch(4); + + private int code; + + SyncType(int code) { + this.code = code; + } + + public int getCode() { + return code; + } +} diff --git a/qmq-tools/pom.xml b/qmq-tools/pom.xml new file mode 100644 index 00000000..71cf8c76 --- /dev/null +++ b/qmq-tools/pom.xml @@ -0,0 +1,33 @@ + + + 4.0.0 + + + qmq + qunar.tc + 4.0.30 + + + qmq-tools + + + + com.ning + async-http-client + + + info.picocli + picocli + + + ch.qos.logback + logback-core + + + ch.qos.logback + logback-classic + + + \ No newline at end of file diff --git a/qmq-tools/src/main/java/qunar/tc/qmq/tools/MetaManagementService.java b/qmq-tools/src/main/java/qunar/tc/qmq/tools/MetaManagementService.java new file mode 100644 index 00000000..2ec6458b --- /dev/null +++ b/qmq-tools/src/main/java/qunar/tc/qmq/tools/MetaManagementService.java @@ -0,0 +1,51 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.tools; + +import com.ning.http.client.AsyncHttpClient; +import com.ning.http.client.Response; + +import java.util.Map; + +/** + * @author keli.wang + * @since 2018-12-05 + */ +public class MetaManagementService { + private final AsyncHttpClient client; + + public MetaManagementService() { + this.client = new AsyncHttpClient(); + } + + public String post(final String metaServer, final String token, final Map params) { + try { + final String url = String.format("http://%s/management", metaServer); + final AsyncHttpClient.BoundRequestBuilder builder = client.preparePost(url); + builder.addHeader("X-Api-Token", token); + params.forEach(builder::addQueryParam); + final Response response = builder.execute().get(); + return response.getResponseBody("UTF-8"); + } catch (Exception e) { + throw new RuntimeException("send request meta server failed.", e); + } + } + + public void close() { + client.close(); + } +} diff --git a/qmq-tools/src/main/java/qunar/tc/qmq/tools/Tools.java b/qmq-tools/src/main/java/qunar/tc/qmq/tools/Tools.java new file mode 100644 index 00000000..07bc823d --- /dev/null +++ b/qmq-tools/src/main/java/qunar/tc/qmq/tools/Tools.java @@ -0,0 +1,50 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.tools; + +import picocli.CommandLine; +import picocli.CommandLine.Command; +import qunar.tc.qmq.tools.command.*; + +/** + * @author keli.wang + * @since 2018-12-05 + */ +@Command(name = "tools.sh", mixinStandardHelpOptions = true) +public class Tools implements Runnable { + public static void main(String[] args) { + final MetaManagementService service = new MetaManagementService(); + + final CommandLine cmd = new CommandLine(new Tools()); + cmd.addSubcommand("AddBroker", new AddBrokerCommand(service)); + cmd.addSubcommand("ReplaceBroker", new ReplaceBrokerCommand(service)); + cmd.addSubcommand("ListBrokers", new ListBrokersCommand(service)); + cmd.addSubcommand("ListBrokerGroups", new ListBrokerGroupsCommand(service)); + cmd.addSubcommand("ListSubjectRoutes", new ListSubjectRoutesCommand(service)); + cmd.addSubcommand("AddSubjectBrokerGroup", new AddSubjectBrokerGroupCommand(service)); + cmd.addSubcommand("RemoveSubjectBrokerGroup", new RemoveSubjectBrokerGroupCommand(service)); + cmd.addSubcommand("AddNewSubject", new AddNewSubjectCommand(service)); + cmd.addSubcommand("ExtendSubjectRoute", new ExtendSubjectRouteCommand(service)); + cmd.parseWithHandler(new CommandLine.RunLast(), args); + service.close(); + } + + @Override + public void run() { + CommandLine.usage(this, System.out); + } +} diff --git a/qmq-tools/src/main/java/qunar/tc/qmq/tools/command/AddBrokerCommand.java b/qmq-tools/src/main/java/qunar/tc/qmq/tools/command/AddBrokerCommand.java new file mode 100644 index 00000000..1b2db144 --- /dev/null +++ b/qmq-tools/src/main/java/qunar/tc/qmq/tools/command/AddBrokerCommand.java @@ -0,0 +1,74 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.tools.command; + +import picocli.CommandLine.Command; +import picocli.CommandLine.Option; +import qunar.tc.qmq.tools.MetaManagementService; + +import java.util.HashMap; + +/** + * @author keli.wang + * @since 2018-12-05 + */ +@Command(name = "AddBroker", mixinStandardHelpOptions = true, sortOptions = false) +public class AddBrokerCommand implements Runnable { + private final MetaManagementService service; + + @Option(names = {"--metaserver"}, required = true, description = {"meta server address, format: or :"}) + private String metaserver; + + @Option(names = {"--token"}, required = true) + private String apiToken; + + @Option(names = {"--brokerGroup"}, required = true) + private String brokerGroup; + + @Option(names = {"--role"}, required = true) + private String role; + + @Option(names = {"--hostname"}, required = true) + private String hostname; + + @Option(names = {"--ip"}, required = true) + private String ip; + + @Option(names = {"--servePort"}, required = true) + private int servePort; + + @Option(names = {"--syncPort"}, required = true) + private int syncPort; + + public AddBrokerCommand(final MetaManagementService service) { + this.service = service; + } + + @Override + public void run() { + final HashMap params = new HashMap<>(); + params.put("action", "AddBroker"); + params.put("brokerGroup", brokerGroup); + params.put("role", role); + params.put("hostname", hostname); + params.put("ip", ip); + params.put("servePort", Integer.toString(servePort)); + params.put("syncPort", Integer.toString(syncPort)); + + System.out.println(service.post(metaserver, apiToken, params)); + } +} diff --git a/qmq-tools/src/main/java/qunar/tc/qmq/tools/command/AddNewSubjectCommand.java b/qmq-tools/src/main/java/qunar/tc/qmq/tools/command/AddNewSubjectCommand.java new file mode 100644 index 00000000..1a463664 --- /dev/null +++ b/qmq-tools/src/main/java/qunar/tc/qmq/tools/command/AddNewSubjectCommand.java @@ -0,0 +1,58 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.tools.command; + +import picocli.CommandLine.Command; +import picocli.CommandLine.Option; +import qunar.tc.qmq.tools.MetaManagementService; + +import java.util.HashMap; + +/** + * @author keli.wang + * @since 2018-12-05 + */ +@Command(name = "AddNewSubject", mixinStandardHelpOptions = true, sortOptions = false) +public class AddNewSubjectCommand implements Runnable { + private final MetaManagementService service; + + @Option(names = {"--metaserver"}, required = true, description = {"meta server address, format: or :"}) + private String metaserver; + + @Option(names = {"--token"}, required = true) + private String apiToken; + + @Option(names = {"--subject"}, required = true) + private String subject; + + @Option(names = {"--tag"}, required = true) + private String tag; + + public AddNewSubjectCommand(final MetaManagementService service) { + this.service = service; + } + + @Override + public void run() { + final HashMap params = new HashMap<>(); + params.put("action", "AddNewSubject"); + params.put("subject", subject); + params.put("tag", tag); + + System.out.println(service.post(metaserver, apiToken, params)); + } +} diff --git a/qmq-tools/src/main/java/qunar/tc/qmq/tools/command/AddSubjectBrokerGroupCommand.java b/qmq-tools/src/main/java/qunar/tc/qmq/tools/command/AddSubjectBrokerGroupCommand.java new file mode 100644 index 00000000..8d74a37d --- /dev/null +++ b/qmq-tools/src/main/java/qunar/tc/qmq/tools/command/AddSubjectBrokerGroupCommand.java @@ -0,0 +1,58 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.tools.command; + +import picocli.CommandLine.Command; +import picocli.CommandLine.Option; +import qunar.tc.qmq.tools.MetaManagementService; + +import java.util.HashMap; + +/** + * @author keli.wang + * @since 2018-12-05 + */ +@Command(name = "AddSubjectBrokerGroup", mixinStandardHelpOptions = true, sortOptions = false) +public class AddSubjectBrokerGroupCommand implements Runnable { + private final MetaManagementService service; + + @Option(names = {"--metaserver"}, required = true, description = {"meta server address, format: or :"}) + private String metaserver; + + @Option(names = {"--token"}, required = true) + private String apiToken; + + @Option(names = {"--brokerGroup"}, required = true) + private String brokerGroup; + + @Option(names = {"--subject"}, required = true) + private String subject; + + public AddSubjectBrokerGroupCommand(final MetaManagementService service) { + this.service = service; + } + + @Override + public void run() { + final HashMap params = new HashMap<>(); + params.put("action", "AddSubjectBrokerGroup"); + params.put("brokerGroup", brokerGroup); + params.put("subject", subject); + + System.out.println(service.post(metaserver, apiToken, params)); + } +} diff --git a/qmq-tools/src/main/java/qunar/tc/qmq/tools/command/ExtendSubjectRouteCommand.java b/qmq-tools/src/main/java/qunar/tc/qmq/tools/command/ExtendSubjectRouteCommand.java new file mode 100644 index 00000000..53b84549 --- /dev/null +++ b/qmq-tools/src/main/java/qunar/tc/qmq/tools/command/ExtendSubjectRouteCommand.java @@ -0,0 +1,62 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.tools.command; + +import picocli.CommandLine.Command; +import picocli.CommandLine.Option; +import qunar.tc.qmq.tools.MetaManagementService; + +import java.util.HashMap; + +/** + * @author keli.wang + * @since 2018-12-05 + */ +@Command(name = "ExtendSubjectRoute", mixinStandardHelpOptions = true, sortOptions = false) +public class ExtendSubjectRouteCommand implements Runnable { + private final MetaManagementService service; + + @Option(names = {"--metaserver"}, required = true, description = {"meta server address, format: or :"}) + private String metaserver; + + @Option(names = {"--token"}, required = true) + private String apiToken; + + @Option(names = {"--relatedSubject"}, required = true, description = {"one subject or *, * means all subject"}) + private String relatedSubject; + + @Option(names = {"--relatedBrokerGroup"}, required = true) + private String relatedBrokerGroup; + + @Option(names = {"--newBrokerGroup"}, required = true) + private String newBrokerGroup; + + public ExtendSubjectRouteCommand(final MetaManagementService service) { + this.service = service; + } + + @Override + public void run() { + final HashMap params = new HashMap<>(); + params.put("action", "ExtendSubjectRoute"); + params.put("relatedSubject", relatedSubject); + params.put("relatedBrokerGroup", relatedBrokerGroup); + params.put("newBrokerGroup", newBrokerGroup); + + System.out.println(service.post(metaserver, apiToken, params)); + } +} diff --git a/qmq-tools/src/main/java/qunar/tc/qmq/tools/command/ListBrokerGroupsCommand.java b/qmq-tools/src/main/java/qunar/tc/qmq/tools/command/ListBrokerGroupsCommand.java new file mode 100644 index 00000000..43609439 --- /dev/null +++ b/qmq-tools/src/main/java/qunar/tc/qmq/tools/command/ListBrokerGroupsCommand.java @@ -0,0 +1,47 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.tools.command; + +import picocli.CommandLine.Command; +import picocli.CommandLine.Option; +import qunar.tc.qmq.tools.MetaManagementService; + +import java.util.HashMap; + +/** + * @author keli.wang + * @since 2018-12-05 + */ +@Command(name = "ListBrokerGroups", mixinStandardHelpOptions = true, sortOptions = false) +public class ListBrokerGroupsCommand implements Runnable { + private final MetaManagementService service; + + @Option(names = {"--metaserver"}, required = true, description = {"meta server address, format: or :"}) + private String metaserver; + + public ListBrokerGroupsCommand(final MetaManagementService service) { + this.service = service; + } + + @Override + public void run() { + final HashMap params = new HashMap<>(); + params.put("action", "ListBrokerGroups"); + + System.out.println(service.post(metaserver, "", params)); + } +} diff --git a/qmq-tools/src/main/java/qunar/tc/qmq/tools/command/ListBrokersCommand.java b/qmq-tools/src/main/java/qunar/tc/qmq/tools/command/ListBrokersCommand.java new file mode 100644 index 00000000..5dff6023 --- /dev/null +++ b/qmq-tools/src/main/java/qunar/tc/qmq/tools/command/ListBrokersCommand.java @@ -0,0 +1,51 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.tools.command; + +import picocli.CommandLine.Command; +import picocli.CommandLine.Option; +import qunar.tc.qmq.tools.MetaManagementService; + +import java.util.HashMap; + +/** + * @author keli.wang + * @since 2018-12-05 + */ +@Command(name = "ListBrokers", mixinStandardHelpOptions = true, sortOptions = false) +public class ListBrokersCommand implements Runnable { + private final MetaManagementService service; + + @Option(names = {"--metaserver"}, required = true, description = {"meta server address, format: or :"}) + private String metaserver; + + @Option(names = {"--brokerGroup"}) + private String brokerGroup; + + public ListBrokersCommand(final MetaManagementService service) { + this.service = service; + } + + @Override + public void run() { + final HashMap params = new HashMap<>(); + params.put("action", "ListBrokers"); + params.put("brokerGroup", brokerGroup == null ? "" : brokerGroup); + + System.out.println(service.post(metaserver, "", params)); + } +} diff --git a/qmq-tools/src/main/java/qunar/tc/qmq/tools/command/ListSubjectRoutesCommand.java b/qmq-tools/src/main/java/qunar/tc/qmq/tools/command/ListSubjectRoutesCommand.java new file mode 100644 index 00000000..84aa7697 --- /dev/null +++ b/qmq-tools/src/main/java/qunar/tc/qmq/tools/command/ListSubjectRoutesCommand.java @@ -0,0 +1,47 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.tools.command; + +import picocli.CommandLine.Command; +import picocli.CommandLine.Option; +import qunar.tc.qmq.tools.MetaManagementService; + +import java.util.HashMap; + +/** + * @author keli.wang + * @since 2018-12-05 + */ +@Command(name = "ListSubjectRoutes", mixinStandardHelpOptions = true, sortOptions = false) +public class ListSubjectRoutesCommand implements Runnable { + private final MetaManagementService service; + + @Option(names = {"--metaserver"}, required = true, description = {"meta server address, format: or :"}) + private String metaserver; + + public ListSubjectRoutesCommand(final MetaManagementService service) { + this.service = service; + } + + @Override + public void run() { + final HashMap params = new HashMap<>(); + params.put("action", "ListSubjectRoutes"); + + System.out.println(service.post(metaserver, "", params)); + } +} diff --git a/qmq-tools/src/main/java/qunar/tc/qmq/tools/command/RemoveSubjectBrokerGroupCommand.java b/qmq-tools/src/main/java/qunar/tc/qmq/tools/command/RemoveSubjectBrokerGroupCommand.java new file mode 100644 index 00000000..5686570d --- /dev/null +++ b/qmq-tools/src/main/java/qunar/tc/qmq/tools/command/RemoveSubjectBrokerGroupCommand.java @@ -0,0 +1,58 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.tools.command; + +import picocli.CommandLine.Command; +import picocli.CommandLine.Option; +import qunar.tc.qmq.tools.MetaManagementService; + +import java.util.HashMap; + +/** + * @author keli.wang + * @since 2018-12-05 + */ +@Command(name = "RemoveSubjectBrokerGroup", mixinStandardHelpOptions = true, sortOptions = false) +public class RemoveSubjectBrokerGroupCommand implements Runnable { + private final MetaManagementService service; + + @Option(names = {"--metaserver"}, required = true, description = {"meta server address, format: or :"}) + private String metaserver; + + @Option(names = {"--token"}, required = true) + private String apiToken; + + @Option(names = {"--brokerGroup"}, required = true) + private String brokerGroup; + + @Option(names = {"--subject"}, required = true, description = {"one subject or *, * means all subjects"}) + private String subject; + + public RemoveSubjectBrokerGroupCommand(final MetaManagementService service) { + this.service = service; + } + + @Override + public void run() { + final HashMap params = new HashMap<>(); + params.put("action", "RemoveSubjectBrokerGroup"); + params.put("brokerGroup", brokerGroup); + params.put("subject", subject); + + System.out.println(service.post(metaserver, apiToken, params)); + } +} diff --git a/qmq-tools/src/main/java/qunar/tc/qmq/tools/command/ReplaceBrokerCommand.java b/qmq-tools/src/main/java/qunar/tc/qmq/tools/command/ReplaceBrokerCommand.java new file mode 100644 index 00000000..1798f9c1 --- /dev/null +++ b/qmq-tools/src/main/java/qunar/tc/qmq/tools/command/ReplaceBrokerCommand.java @@ -0,0 +1,75 @@ +/* + * Copyright 2018 Qunar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License.com.qunar.pay.trade.api.card.service.usercard.UserCardQueryFacade + */ + +package qunar.tc.qmq.tools.command; + +import picocli.CommandLine.Command; +import picocli.CommandLine.Option; +import qunar.tc.qmq.tools.MetaManagementService; + +import java.util.HashMap; + +/** + * @author keli.wang + * @since 2018-12-05 + */ +@Command(name = "ReplaceBroker", mixinStandardHelpOptions = true, sortOptions = false) +public class ReplaceBrokerCommand implements Runnable { + private final MetaManagementService service; + + @Option(names = {"--metaserver"}, required = true, description = {"meta server address, format: or :"}) + private String metaserver; + + @Option(names = {"--token"}, required = true) + private String apiToken; + + @Option(names = {"--brokerGroup"}, required = true) + private String brokerGroup; + + @Option(names = {"--role"}, required = true) + private String role; + + @Option(names = {"--hostname"}, required = true) + private String hostname; + + @Option(names = {"--ip"}, required = true) + private String ip; + + @Option(names = {"--servePort"}, required = true) + private int servePort; + + @Option(names = {"--syncPort"}, required = true) + private int syncPort; + + + public ReplaceBrokerCommand(final MetaManagementService service) { + this.service = service; + } + + @Override + public void run() { + final HashMap params = new HashMap<>(); + params.put("action", "ReplaceBroker"); + params.put("brokerGroup", brokerGroup); + params.put("role", role); + params.put("hostname", hostname); + params.put("ip", ip); + params.put("servePort", Integer.toString(servePort)); + params.put("syncPort", Integer.toString(syncPort)); + + System.out.println(service.post(metaserver, apiToken, params)); + } +}