Skip to content

Commit

Permalink
GEODE-7090 Remove dependency on DataSerializer from membership classes
Browse files Browse the repository at this point in the history
DataSerializableFixedID is used by most internal classes for
serialization.  Membership, in particular, was constrained in its
implementation to use only "fixed" serialization identifiers because our
other serialization mechanisms (DataSerialiable, PDX) are dependent on
membership and messaging.

This PR moves serialization of DataSerializableFixedID to a new
sub-project, geode-serialization in support of also separating
membership into its own sub-project.

Many files are touched by this PR.  The vast majority are due to a
change in the serialization interface that introduces a new parameter in
DataSerializableFixedID's toData and fromData methods.  This parameter
gives access to the serializer and to the Version of the
source/destination.  This parameter was added to begin moving from a
"static" based serialization service to an instance based service.

Serialization code concerning "DSFID"s has been moved from
InternalDataSerializer to the new class DSFIDSerialzerImpl.
InternalDataSerializer holds on to an instance of this class and
delegates DSFID serialization to it.

InternalDataSerializer also effects registration of Geode's DSFID
classes with this serializer.  Membership "Services" handles
registration of its own DSFID classes with the service, as do other
sub-projects like geode-web.

HeapDataOutputStream has been used by most of our serialization code to
provide a DataOutput for our toData methods.  Over the years this class
has accumulated a lot of adornments for PDX serialization that aren't
appropriate for a geode-serialization sub-project.  Consequently I've
moved much of its code into a superclass, BufferDataOutputStream, that
provides most of its functionality w/o needing to reference other geode-
core classes.

Version.java has been repackaged in this PR, moving it into a
"serialization" package.  This affected some rolling-upgrade and
backward-compatibility tests that were referring to
Version.CURRENT_ORDINAL.  These have been modified to get the "current
version ordinal" from VersionManager.

Finally, InternalDataSerializer injects a serializer into the new
sub-project that provideds an API for handling other types of
serializable objects like PDX.  This serializer is available in the
DataSerializableFixedID toData/fromData context parameter and is used by
membership services when serializing Geode DistributionMessages.
  • Loading branch information
bschuchardt committed Sep 9, 2019
1 parent e1f200e commit 6ee6d96
Show file tree
Hide file tree
Showing 693 changed files with 9,149 additions and 5,604 deletions.
8 changes: 7 additions & 1 deletion boms/geode-all-bom/src/test/resources/expected-pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -871,10 +871,16 @@
<version>1.11.0-SNAPSHOT</version>
<scope>compile</scope>
</dependency>
<dependency>
<groupId>org.apache.geode</groupId>
<artifactId>geode-serialization</artifactId>
<version>1.11.0-SNAPSHOT</version>
<scope>compile</scope>
</dependency>
<dependency>
<groupId>org.apache.geode</groupId>
<artifactId>geode-unsafe</artifactId>
<version>1.10.0-SNAPSHOT</version>
<version>1.11.0-SNAPSHOT</version>
<scope>compile</scope>
</dependency>
<dependency>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -262,6 +262,7 @@ SESSION_JAR="${LIB_DIR}/geode-modules-session-${VERSION}.jar"

declare -a OTHER_JARS
OTHER_JARS=(${GEODE}/lib/geode-core-${VERSION}.jar \
${GEODE}/lib/geode-serialization-${VERSION}.jar \
${GEODE}/lib/geode-common-${VERSION}.jar \
${GEODE}/lib/geode-management-${VERSION}.jar \
${GEODE}/lib/antlr-@[email protected] \
Expand Down
2 changes: 2 additions & 0 deletions geode-assembly/build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@ import java.nio.file.Paths
def dependentProjectNames = [
':geode-common',
':geode-connectors',
':geode-serialization',
':geode-core',
':geode-cq',
':geode-lucene',
Expand Down Expand Up @@ -167,6 +168,7 @@ dependencies {
geodeArchives project(':geode-protobuf-messages')
geodeArchives project(':geode-rebalancer')
geodeArchives project(':geode-redis')
geodeArchives project(':geode-serialization')
geodeArchives project(':geode-wan')
geodeArchives project(':geode-web')
geodeArchives project(':geode-web-api')
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -91,7 +91,8 @@ public String jarSkipPropertyName() {

private static final String[] tomcatRequiredJars =
{"antlr", "commons-io", "commons-lang", "commons-validator", "fastutil", "geode-common",
"geode-core", "geode-management", "javax.transaction-api", "jgroups", "log4j-api",
"geode-core", "geode-management", "geode-serialization", "javax.transaction-api",
"jgroups", "log4j-api",
"log4j-core", "log4j-jul", "micrometer", "shiro-core", "jetty-server", "jetty-util",
"jetty-http", "jetty-io"};

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -954,6 +954,7 @@ lib/geode-protobuf-0.0.0.jar
lib/geode-protobuf-messages-0.0.0.jar
lib/geode-rebalancer-0.0.0.jar
lib/geode-redis-0.0.0.jar
lib/geode-serialization-0.0.0.jar
lib/geode-unsafe-0.0.0.jar
lib/geode-wan-0.0.0.jar
lib/gfsh-dependencies.jar
Expand Down
121 changes: 61 additions & 60 deletions geode-assembly/src/integrationTest/resources/dependency_classpath.txt
Original file line number Diff line number Diff line change
@@ -1,84 +1,85 @@
HdrHistogram-2.1.11.jar
HikariCP-3.2.0.jar
LatencyUtils-2.0.3.jar
antlr-2.7.7.jar
classgraph-4.0.6.jar
commons-beanutils-1.9.3.jar
commons-codec-1.10.jar
commons-collections-3.2.2.jar
commons-io-2.6.jar
commons-lang3-3.8.1.jar
commons-logging-1.2.jar
commons-math3-3.2.jar
commons-validator-1.6.jar
fastutil-8.2.2.jar
geo-0.7.1.jar
geode-common-0.0.0.jar
geode-unsafe-0.0.0.jar
geode-connectors-0.0.0.jar
geode-core-0.0.0.jar
geode-cq-0.0.0.jar
geode-lucene-0.0.0.jar
geode-redis-0.0.0.jar
geode-memcached-0.0.0.jar
geode-management-0.0.0.jar
geode-memcached-0.0.0.jar
geode-old-client-support-0.0.0.jar
geode-protobuf-0.0.0.jar
geode-protobuf-messages-0.0.0.jar
geode-rebalancer-0.0.0.jar
geode-redis-0.0.0.jar
geode-serialization-0.0.0.jar
geode-wan-0.0.0.jar
grumpy-core-0.2.2.jar
httpclient-4.5.6.jar
httpcore-4.4.10.jar
istack-commons-runtime-2.2.jar
jackson-databind-2.9.8.jar
jackson-annotations-2.9.8.jar
jackson-core-2.9.8.jar
jackson-databind-2.9.8.jar
jansi-1.17.1.jar
javax.activation-1.2.0.jar
javax.activation-api-1.2.0.jar
javax.resource-api-1.7.1.jar
javax.servlet-api-3.1.0.jar
javax.transaction-api-1.3.jar
geode-unsafe-0.0.0.jar
HikariCP-3.2.0.jar
commons-lang3-3.8.1.jar
jaxb-api-2.3.1.jar
log4j-jcl-2.12.0.jar
log4j-slf4j-impl-2.12.0.jar
log4j-core-2.12.0.jar
log4j-jul-2.12.0.jar
log4j-api-2.12.0.jar
spring-shell-1.2.0.RELEASE.jar
rmiio-2.1.2.jar
jaxb-impl-2.3.1.jar
jetty-http-9.4.12.v20180830.jar
jetty-io-9.4.12.v20180830.jar
antlr-2.7.7.jar
javax.activation-1.2.0.jar
istack-commons-runtime-2.2.jar
commons-validator-1.6.jar
shiro-core-1.4.0.jar
shiro-config-ogdl-1.4.0.jar
commons-beanutils-1.9.3.jar
commons-collections-3.2.2.jar
commons-io-2.6.jar
spring-core-4.3.23.RELEASE.jar
httpclient-4.5.6.jar
commons-logging-1.2.jar
classgraph-4.0.6.jar
micrometer-core-1.2.0.jar
fastutil-8.2.2.jar
javax.resource-api-1.7.1.jar
jetty-webapp-9.4.12.v20180830.jar
jetty-servlet-9.4.12.v20180830.jar
jetty-security-9.4.12.v20180830.jar
jetty-server-9.4.12.v20180830.jar
jetty-servlet-9.4.12.v20180830.jar
jetty-util-9.4.12.v20180830.jar
jetty-webapp-9.4.12.v20180830.jar
jetty-xml-9.4.12.v20180830.jar
jgroups-3.6.14.Final.jar
jline-2.12.jar
javax.servlet-api-3.1.0.jar
jna-4.1.0.jar
jopt-simple-5.0.4.jar
log4j-api-2.12.0.jar
log4j-core-2.12.0.jar
log4j-jcl-2.12.0.jar
log4j-jul-2.12.0.jar
log4j-slf4j-impl-2.12.0.jar
lucene-analyzers-common-6.6.2.jar
lucene-analyzers-phonetic-6.6.2.jar
lucene-core-6.6.2.jar
lucene-queries-6.6.2.jar
lucene-queryparser-6.6.2.jar
micrometer-core-1.2.0.jar
netty-all-4.1.31.Final.jar
protobuf-java-3.6.1.jar
rmiio-2.1.2.jar
httpcore-4.4.10.jar
jansi-1.17.1.jar
snappy-0.4.jar
jgroups-3.6.14.Final.jar
shiro-cache-1.4.0.jar
shiro-config-core-1.4.0.jar
shiro-config-ogdl-1.4.0.jar
shiro-core-1.4.0.jar
shiro-crypto-cipher-1.4.0.jar
shiro-crypto-core-1.4.0.jar
shiro-crypto-hash-1.4.0.jar
shiro-crypto-cipher-1.4.0.jar
shiro-config-core-1.4.0.jar
shiro-event-1.4.0.jar
shiro-crypto-core-1.4.0.jar
shiro-lang-1.4.0.jar
slf4j-api-1.7.25.jar
snappy-0.4.jar
spring-core-4.3.23.RELEASE.jar
spring-shell-1.2.0.RELEASE.jar
swagger-annotations-1.5.20.jar
swagger-annotations-1.5.20.jar
javax.activation-api-1.2.0.jar
jline-2.12.jar
HdrHistogram-2.1.11.jar
LatencyUtils-2.0.3.jar
javax.transaction-api-1.3.jar
jetty-xml-9.4.12.v20180830.jar
jetty-http-9.4.12.v20180830.jar
jetty-io-9.4.12.v20180830.jar
jetty-util-9.4.12.v20180830.jar
commons-codec-1.10.jar
lucene-analyzers-phonetic-6.6.2.jar
lucene-analyzers-common-6.6.2.jar
lucene-queryparser-6.6.2.jar
lucene-core-6.6.2.jar
lucene-queries-6.6.2.jar
protobuf-java-3.6.1.jar
geo-0.7.1.jar
netty-all-4.1.31.Final.jar
grumpy-core-0.2.2.jar
commons-math3-3.2.jar
3 changes: 3 additions & 0 deletions geode-core/build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -332,6 +332,8 @@ dependencies {
//probably does not belong in this project
api(project(':geode-common'))
api(project(':geode-unsafe'))

api(project(':geode-serialization'))

//geode-management currently has pieces of the public API
//copied into it, so it is an API dependency
Expand Down Expand Up @@ -362,6 +364,7 @@ dependencies {
testRuntime('commons-io:commons-io')
testRuntime('commons-validator:commons-validator')
testRuntime('com.pholser:junit-quickcheck-generators')
testRuntime(project(':geode-old-versions'))


integrationTestCompile(project(':geode-junit')) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@
import static org.apache.geode.distributed.ConfigurationProperties.SSL_TRUSTSTORE;
import static org.apache.geode.distributed.ConfigurationProperties.SSL_TRUSTSTORE_PASSWORD;
import static org.apache.geode.distributed.ConfigurationProperties.USE_CLUSTER_CONFIGURATION;
import static org.apache.geode.internal.DataSerializableFixedID.SERIAL_ACKED_MESSAGE;
import static org.apache.geode.internal.serialization.DataSerializableFixedID.SERIAL_ACKED_MESSAGE;
import static org.apache.geode.test.awaitility.GeodeAwaitility.await;
import static org.apache.geode.test.awaitility.GeodeAwaitility.getTimeout;
import static org.apache.geode.test.dunit.IgnoredException.addIgnoredException;
Expand Down Expand Up @@ -77,8 +77,10 @@
import org.apache.geode.distributed.internal.SerialAckedMessage;
import org.apache.geode.distributed.internal.membership.gms.membership.GMSJoinLeave;
import org.apache.geode.internal.AvailablePortHelper;
import org.apache.geode.internal.DSFIDFactory;
import org.apache.geode.internal.InternalDataSerializer;
import org.apache.geode.internal.cache.DirectReplyMessage;
import org.apache.geode.internal.serialization.DeserializationContext;
import org.apache.geode.internal.serialization.SerializationContext;
import org.apache.geode.test.dunit.DistributedTestUtils;
import org.apache.geode.test.dunit.Host;
import org.apache.geode.test.dunit.VM;
Expand Down Expand Up @@ -172,8 +174,9 @@ public void createEntryWithBigMessage() {

@Test
public void receiveBigResponse() {
invokeInEveryVM(() -> DSFIDFactory.registerDSFID(SERIAL_ACKED_MESSAGE,
SerialAckedMessageWithBigReply.class));
invokeInEveryVM(
() -> InternalDataSerializer.getDSFIDSerializer().registerDSFID(SERIAL_ACKED_MESSAGE,
SerialAckedMessageWithBigReply.class));
try {
int locatorPort = createLocator(getVM(0));
for (int i = 1; i <= NUM_SERVERS; i++) {
Expand All @@ -190,7 +193,8 @@ public void receiveBigResponse() {
});
} finally {
invokeInEveryVM(
() -> DSFIDFactory.registerDSFID(SERIAL_ACKED_MESSAGE, SerialAckedMessage.class));
() -> InternalDataSerializer.getDSFIDSerializer().registerDSFID(SERIAL_ACKED_MESSAGE,
SerialAckedMessage.class));
}
}

Expand Down Expand Up @@ -393,14 +397,16 @@ public void send(Set<DistributedMember> recipients)
}

@Override
public void toData(DataOutput out) throws IOException {
super.toData(out);
public void toData(DataOutput out,
SerializationContext context) throws IOException {
super.toData(out, context);
out.writeInt(processorId);
}

@Override
public void fromData(DataInput in) throws IOException, ClassNotFoundException {
super.fromData(in);
public void fromData(DataInput in,
DeserializationContext context) throws IOException, ClassNotFoundException {
super.fromData(in, context);
processorId = in.readInt();
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,6 @@
import org.apache.geode.distributed.internal.membership.gms.GMSMember;
import org.apache.geode.distributed.internal.membership.gms.MembershipManagerHelper;
import org.apache.geode.internal.HeapDataOutputStream;
import org.apache.geode.internal.Version;
import org.apache.geode.test.dunit.Host;
import org.apache.geode.test.dunit.SerializableCallable;
import org.apache.geode.test.dunit.SerializableRunnable;
Expand Down Expand Up @@ -270,7 +269,7 @@ private InternalDistributedMember connectAndSetUpPartialID() throws Exception {
assertEquals(gmsMember,
((GMSMemberAdapter) internalDistributedMember.getNetMember()).getGmsMember());
gmsMember.setName(null);
HeapDataOutputStream outputStream = new HeapDataOutputStream(100, Version.CURRENT);
HeapDataOutputStream outputStream = new HeapDataOutputStream(100);
new InternalDistributedMember(new GMSMemberAdapter(gmsMember)).writeEssentialData(outputStream);
DataInputStream dataInputStream =
new DataInputStream(new ByteArrayInputStream(outputStream.toByteArray()));
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -236,17 +236,16 @@ public void alert(Alert alert) {
}

private void populateCache() {
AttributesFactory attributesFactory = new AttributesFactory();
attributesFactory.setScope(Scope.DISTRIBUTED_NO_ACK);

RegionAttributes regionAttributes = attributesFactory.create();

for (int i = 0; i < Host.getHostCount(); i++) {
Host host = Host.getHost(i);

for (int j = 0; j < host.getVMCount(); j++) {
VM vm = host.getVM(j);
vm.invoke(() -> {
AttributesFactory attributesFactory = new AttributesFactory();
attributesFactory.setScope(Scope.DISTRIBUTED_NO_ACK);

RegionAttributes regionAttributes = attributesFactory.create();
createRegion("cdm-testSubRegion1", regionAttributes);
createRegion("cdm-testSubRegion2", regionAttributes);
createRegion("cdm-testSubRegion3", regionAttributes);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,9 @@

import org.apache.geode.DataSerializer;
import org.apache.geode.internal.Assert;
import org.apache.geode.internal.Version;
import org.apache.geode.internal.serialization.DeserializationContext;
import org.apache.geode.internal.serialization.SerializationContext;
import org.apache.geode.internal.serialization.Version;

/**
* This message simply contains a date
Expand Down Expand Up @@ -78,15 +80,17 @@ public int getDSFID() {
}

@Override
public void toData(DataOutput out) throws IOException {
super.toData(out);
public void toData(DataOutput out,
SerializationContext context) throws IOException {
super.toData(out, context);
DataSerializer.writeObject(this.date, out);
}

@Override
public void fromData(DataInput in) throws IOException, ClassNotFoundException {
public void fromData(DataInput in,
DeserializationContext context) throws IOException, ClassNotFoundException {

super.fromData(in);
super.fromData(in, context);
this.date = (Date) DataSerializer.readObject(in);
}

Expand Down
Loading

0 comments on commit 6ee6d96

Please sign in to comment.