I'm using JGit in a webservice, that needs to pull/fetch from multiple repositories. Therefore, I'm using the InMemoryRepository of JGit. Additionally, I want to access the repo using SSH (private/public RSA key pair).
As described in https://bugs.eclipse.org/bugs/show_bug.cgi?id=549456 it seems that this is not possible. I found another post, where Thomas Wolf describes how this could be achieved.
Somehow this doesn't work for me. So far this is my code.
private void fetchGitRepository(String repositoryUri, java.security.KeyPair keyPair) {
DfsRepositoryDescription repoDesc = new DfsRepositoryDescription();
Repository repository = new InMemoryRepository(repoDesc);
repository.getObjectDatabase();
try (Git git = new Git(repository)) {
FetchCommand fetchCommand = git.fetch()
.setRemote(repositoryUri)
.setRefSpecs(new RefSpec("+refs/heads/*:refs/heads/*"));
if (keyPair != null) {
SshTransportConfigCallback transportConfigCallback = new SshTransportConfigCallback(keyPair);
fetchCommand.setTransportConfigCallback(transportConfigCallback);
}
fetchCommand.call();
}
}
class SshTransportConfigCallback implements TransportConfigCallback {
private KeyPair keyPair;
public SshTransportConfigCallback(KeyPair keyPair) {
this.keyPair = keyPair;
}
#Override
public void configure(Transport transport) {
if (transport instanceof SshTransport) {
SshTransport sshTransport = (SshTransport) transport;
sshTransport.setSshSessionFactory(new MySessionFactory(keyPair));
}
}
private static class MySessionFactory extends SshdSessionFactory {
private final KeyPair keyPair;
public MySessionFactory(KeyPair keyPair) {
this.keyPair = keyPair;
}
#Override
public File getHomeDirectory() {
return null;
}
#Override
public File getSshDirectory() {
return null;
}
#Override
protected File getSshConfig(File sshDir) {
return null;
}
#Override
protected ServerKeyDatabase getServerKeyDatabase(File homeDir, File sshDir) {
return new ServerKeyDatabase() {
#Override
public List<PublicKey> lookup(String connectAddress, InetSocketAddress remoteAddress, Configuration config) {
return emptyList();
}
#Override
public boolean accept(String connectAddress, InetSocketAddress remoteAddress, PublicKey serverKey, Configuration config, CredentialsProvider provider) {
return true; // TODO: Fix this later
}
};
}
#Override
protected List<Path> getDefaultKnownHostsFiles(File sshDir) {
return emptyList();
}
#Override
protected Iterable<KeyPair> getDefaultKeys(File sshDir) {
return of(keyPair);
}
#Override
protected List<Path> getDefaultIdentities(File sshDir) {
return emptyList();
}
#Override
protected String getDefaultPreferredAuthentications() {
return "publickey";
}
}
}
I'm getting the following exception:
2021-10-23 18:04:49.773 DEBUG 25750 --- [nio-9000-exec-5] o.a.sshd.common.io.nio2.Nio2Connector : Connecting to github.com/140.82.121.3:22
2021-10-23 18:04:49.774 DEBUG 25750 --- [nio-9000-exec-5] o.a.sshd.common.io.nio2.Nio2Connector : setOption(SO_REUSEADDR)[true] from property=Property[socket-reuseaddr](Boolean]
2021-10-23 18:04:49.801 DEBUG 25750 --- []-nio2-thread-1] o.a.sshd.common.io.nio2.Nio2Session : Creating IoSession on /192.168.178.37:52145 from github.com/140.82.121.3:22 via null
2021-10-23 18:04:49.801 DEBUG 25750 --- []-nio2-thread-1] o.e.j.i.t.sshd.JGitClientSession : Client session created: Nio2Session[local=/192.168.178.37:52145, remote=github.com/140.82.121.3:22]
2021-10-23 18:04:49.801 DEBUG 25750 --- []-nio2-thread-1] o.a.s.c.session.ClientUserAuthService : ClientUserAuthService(JGitClientSession[null#github.com/140.82.121.3:22]) use configured preferences: publickey
2021-10-23 18:04:49.801 DEBUG 25750 --- []-nio2-thread-1] o.a.s.c.session.ClientUserAuthService : ClientUserAuthService(JGitClientSession[null#github.com/140.82.121.3:22]) client methods: [publickey]
2021-10-23 18:04:49.802 DEBUG 25750 --- []-nio2-thread-1] o.a.s.c.s.h.SessionTimeoutListener : sessionCreated(JGitClientSession[null#github.com/140.82.121.3:22]) tracking
2021-10-23 18:04:49.802 DEBUG 25750 --- []-nio2-thread-1] o.e.j.i.t.sshd.JGitClientSession : initializeProxyConnector(JGitClientSession[null#github.com/140.82.121.3:22]) no proxy to initialize
2021-10-23 18:04:49.802 DEBUG 25750 --- []-nio2-thread-1] o.e.j.i.t.sshd.JGitClientSession : sendIdentification(JGitClientSession[null#github.com/140.82.121.3:22]): SSH-2.0-APACHE-SSHD-2.7.0
2021-10-23 18:04:49.802 DEBUG 25750 --- []-nio2-thread-1] o.a.sshd.common.io.nio2.Nio2Session : writeBuffer(Nio2Session[local=/192.168.178.37:52145, remote=github.com/140.82.121.3:22]) writing 27 bytes
2021-10-23 18:04:50.375 DEBUG 25750 --- []-nio2-thread-1] o.e.j.i.t.sshd.JGitClientSession : HostKeyAlgorithms [ecdsa-sha2-nistp256-cert-v01#openssh.com, ecdsa-sha2-nistp384-cert-v01#openssh.com, ecdsa-sha2-nistp521-cert-v01#openssh.com, ssh-ed25519-cert-v01#openssh.com, rsa-sha2-512-cert-v01#openssh.com, rsa-sha2-256-cert-v01#openssh.com, ssh-rsa-cert-v01#openssh.com, ecdsa-sha2-nistp256, ecdsa-sha2-nistp384, ecdsa-sha2-nistp521, ssh-ed25519, sk-ecdsa-sha2-nistp256#openssh.com, sk-ssh-ed25519#openssh.com, rsa-sha2-512, rsa-sha2-256, ssh-rsa, ssh-dss-cert-v01#openssh.com, ssh-dss]
2021-10-23 18:04:50.375 DEBUG 25750 --- []-nio2-thread-1] o.e.j.i.t.sshd.JGitClientSession : KexAlgorithms ecdh-sha2-nistp521,ecdh-sha2-nistp384,ecdh-sha2-nistp256,diffie-hellman-group-exchange-sha256,diffie-hellman-group18-sha512,diffie-hellman-group17-sha512,diffie-hellman-group16-sha512,diffie-hellman-group15-sha512,diffie-hellman-group14-sha256,ext-info-c
2021-10-23 18:04:50.375 DEBUG 25750 --- []-nio2-thread-1] s.c.k.e.DefaultClientKexExtensionHandler : handleKexInitProposal(JGitClientSession[null#github.com/140.82.121.3:22]): proposing HostKeyAlgorithms [ecdsa-sha2-nistp256-cert-v01#openssh.com, ecdsa-sha2-nistp384-cert-v01#openssh.com, ecdsa-sha2-nistp521-cert-v01#openssh.com, ssh-ed25519-cert-v01#openssh.com, rsa-sha2-512-cert-v01#openssh.com, rsa-sha2-256-cert-v01#openssh.com, ssh-rsa-cert-v01#openssh.com, ecdsa-sha2-nistp256, ecdsa-sha2-nistp384, ecdsa-sha2-nistp521, ssh-ed25519, sk-ecdsa-sha2-nistp256#openssh.com, sk-ssh-ed25519#openssh.com, rsa-sha2-512, rsa-sha2-256, ssh-rsa, ssh-dss-cert-v01#openssh.com, ssh-dss, ext-info-c]
2021-10-23 18:04:50.375 DEBUG 25750 --- []-nio2-thread-1] o.e.j.i.t.sshd.JGitClientSession : sendKexInit(JGitClientSession[null#github.com/140.82.121.3:22]) Send SSH_MSG_KEXINIT
2021-10-23 18:04:50.375 DEBUG 25750 --- []-nio2-thread-1] o.e.j.i.t.sshd.JGitClientSession : encode(JGitClientSession[null#github.com/140.82.121.3:22]) packet #0 sending command=20[SSH_MSG_KEXINIT] len=1304
2021-10-23 18:04:50.375 DEBUG 25750 --- []-nio2-thread-1] o.a.sshd.common.io.nio2.Nio2Session : writeBuffer(Nio2Session[local=/192.168.178.37:52145, remote=github.com/140.82.121.3:22]) writing 1320 bytes
2021-10-23 18:04:50.375 DEBUG 25750 --- [nio-9000-exec-5] o.a.s.c.future.DefaultConnectFuture : Connected to github.com/140.82.121.3:22 after 601766936 nanos
2021-10-23 18:04:50.375 DEBUG 25750 --- []-nio2-thread-1] o.e.j.i.t.sshd.JGitClientSession : doReadIdentification(JGitClientSession[git#github.com/140.82.121.3:22]) line: SSH-2.0-babeld-abb5d94f
2021-10-23 18:04:50.375 DEBUG 25750 --- []-nio2-thread-1] o.e.j.i.t.sshd.JGitClientSession : readIdentification(JGitClientSession[git#github.com/140.82.121.3:22]) Server version string: SSH-2.0-babeld-abb5d94f
2021-10-23 18:04:50.375 DEBUG 25750 --- [nio-9000-exec-5] o.a.s.c.session.ClientUserAuthService : auth(JGitClientSession[git#github.com/140.82.121.3:22])[ssh-connection] send SSH_MSG_USERAUTH_REQUEST for 'none'
2021-10-23 18:04:50.376 DEBUG 25750 --- []-nio2-thread-1] o.e.j.i.t.sshd.JGitClientSession : handleKexInit(JGitClientSession[git#github.com/140.82.121.3:22]) SSH_MSG_KEXINIT
2021-10-23 18:04:50.376 DEBUG 25750 --- [nio-9000-exec-5] o.e.j.i.t.sshd.JGitClientSession : enqueuePendingPacket(JGitClientSession[git#github.com/140.82.121.3:22])[SSH_MSG_USERAUTH_REQUEST] Start flagging packets as pending until key exchange is done
2021-10-23 18:04:50.376 DEBUG 25750 --- []-nio2-thread-1] o.e.j.i.t.sshd.JGitClientSession : setNegotiationResult(JGitClientSession[git#github.com/140.82.121.3:22]) Kex: server->client aes128-ctr hmac-sha2-256-etm#openssh.com none
2021-10-23 18:04:50.376 DEBUG 25750 --- []-nio2-thread-1] o.e.j.i.t.sshd.JGitClientSession : setNegotiationResult(JGitClientSession[git#github.com/140.82.121.3:22]) Kex: client->server aes128-ctr hmac-sha2-256-etm#openssh.com none
2021-10-23 18:04:50.376 DEBUG 25750 --- []-nio2-thread-1] o.e.j.i.t.sshd.JGitClientSession : setNegotiationResult(JGitClientSession[git#github.com/140.82.121.3:22]) Kex: kex algorithms = ecdh-sha2-nistp521
2021-10-23 18:04:50.376 DEBUG 25750 --- []-nio2-thread-1] o.e.j.i.t.sshd.JGitClientSession : setNegotiationResult(JGitClientSession[git#github.com/140.82.121.3:22]) Kex: server host key algorithms = rsa-sha2-512
2021-10-23 18:04:50.376 DEBUG 25750 --- []-nio2-thread-1] o.e.j.i.t.sshd.JGitClientSession : setNegotiationResult(JGitClientSession[git#github.com/140.82.121.3:22]) Kex: encryption algorithms (client to server) = aes128-ctr
2021-10-23 18:04:50.376 DEBUG 25750 --- []-nio2-thread-1] o.e.j.i.t.sshd.JGitClientSession : setNegotiationResult(JGitClientSession[git#github.com/140.82.121.3:22]) Kex: encryption algorithms (server to client) = aes128-ctr
2021-10-23 18:04:50.376 DEBUG 25750 --- []-nio2-thread-1] o.e.j.i.t.sshd.JGitClientSession : setNegotiationResult(JGitClientSession[git#github.com/140.82.121.3:22]) Kex: mac algorithms (client to server) = hmac-sha2-256-etm#openssh.com
2021-10-23 18:04:50.376 DEBUG 25750 --- []-nio2-thread-1] o.e.j.i.t.sshd.JGitClientSession : setNegotiationResult(JGitClientSession[git#github.com/140.82.121.3:22]) Kex: mac algorithms (server to client) = hmac-sha2-256-etm#openssh.com
2021-10-23 18:04:50.376 DEBUG 25750 --- []-nio2-thread-1] o.e.j.i.t.sshd.JGitClientSession : setNegotiationResult(JGitClientSession[git#github.com/140.82.121.3:22]) Kex: compression algorithms (client to server) = none
2021-10-23 18:04:50.376 DEBUG 25750 --- []-nio2-thread-1] o.e.j.i.t.sshd.JGitClientSession : setNegotiationResult(JGitClientSession[git#github.com/140.82.121.3:22]) Kex: compression algorithms (server to client) = none
2021-10-23 18:04:50.380 DEBUG 25750 --- []-nio2-thread-1] org.apache.sshd.client.kex.DHGClient : init(DHGClient[ecdh-sha2-nistp521])[JGitClientSession[git#github.com/140.82.121.3:22]] Send SSH_MSG_KEXDH_INIT
2021-10-23 18:04:50.380 DEBUG 25750 --- []-nio2-thread-1] o.e.j.i.t.sshd.JGitClientSession : encode(JGitClientSession[git#github.com/140.82.121.3:22]) packet #1 sending command=30[30] len=138
2021-10-23 18:04:50.380 DEBUG 25750 --- []-nio2-thread-1] o.a.sshd.common.io.nio2.Nio2Session : writeBuffer(Nio2Session[local=/192.168.178.37:52145, remote=github.com/140.82.121.3:22]) writing 152 bytes
2021-10-23 18:04:50.495 DEBUG 25750 --- []-nio2-thread-2] org.apache.sshd.client.kex.DHGClient : next(DHGClient[ecdh-sha2-nistp521])[JGitClientSession[git#github.com/140.82.121.3:22]] process command=SSH_MSG_KEXDH_REPLY
2021-10-23 18:04:50.503 DEBUG 25750 --- []-nio2-thread-2] o.e.j.i.t.sshd.JGitClientSession : setServerKey(JGitClientSession[git#github.com/140.82.121.3:22]) keyType=ssh-rsa, digest=SHA256:nThbg6kXUpJWGl7E1IGOCspRomTxdCARLviKw6E5SY8
2021-10-23 18:04:50.503 DEBUG 25750 --- []-nio2-thread-2] o.e.j.i.t.sshd.JGitClientSession : handleKexMessage(JGitClientSession[git#github.com/140.82.121.3:22])[ecdh-sha2-nistp521] KEX processing complete after cmd=31
2021-10-23 18:04:50.995 DEBUG 25750 --- []-nio2-thread-2] o.e.j.i.t.sshd.JGitClientSession : checkKeys(JGitClientSession[git#github.com/140.82.121.3:22]) key=ssh-rsa-SHA256:nThbg6kXUpJWGl7E1IGOCspRomTxdCARLviKw6E5SY8, verified=true
2021-10-23 18:04:50.996 DEBUG 25750 --- []-nio2-thread-2] o.e.j.i.t.sshd.JGitClientSession : sendNewKeys(JGitClientSession[git#github.com/140.82.121.3:22]) Send SSH_MSG_NEWKEYS
2021-10-23 18:04:50.996 DEBUG 25750 --- []-nio2-thread-2] o.e.j.i.t.sshd.JGitClientSession : encode(JGitClientSession[git#github.com/140.82.121.3:22]) packet #2 sending command=21[SSH_MSG_NEWKEYS] len=1
2021-10-23 18:04:50.996 DEBUG 25750 --- []-nio2-thread-2] o.a.sshd.common.io.nio2.Nio2Session : writeBuffer(Nio2Session[local=/192.168.178.37:52145, remote=github.com/140.82.121.3:22]) writing 16 bytes
2021-10-23 18:04:50.996 DEBUG 25750 --- []-nio2-thread-2] o.e.j.i.t.sshd.JGitClientSession : handleNewKeys(JGitClientSession[git#github.com/140.82.121.3:22]) SSH_MSG_NEWKEYS command=SSH_MSG_NEWKEYS
2021-10-23 18:04:50.996 DEBUG 25750 --- []-nio2-thread-2] o.e.j.i.t.sshd.JGitClientSession : receiveNewKeys(JGitClientSession[git#github.com/140.82.121.3:22]) session ID=37:31:1f:99:1c:de:35:10:3c:e7:31:be:39:5b:ca:b0:5b:c0:e6:89:92:9c:6e:c2:cf:77:e9:a6:12:a7:3b:aa:ca:12:61:1e:43:b1:a4:d3:7e:30:a9:98:43:c4:76:b3:fd:f9:2c:a7:1d:e9:de:1c:c5:b4:90:ca:0e:c5:78:94
2021-10-23 18:04:50.997 DEBUG 25750 --- []-nio2-thread-2] o.e.j.i.t.sshd.JGitClientSession : receiveNewKeys(JGitClientSession[git#github.com/140.82.121.3:22]) inCipher=BaseCipher[AES, ivSize=16, kdfSize=16,AES/CTR/NoPadding, blkSize=16], outCipher=BaseCipher[AES, ivSize=16, kdfSize=16,AES/CTR/NoPadding, blkSize=16], recommended blocks limit=4294967296, actual=4294967296
2021-10-23 18:04:50.997 DEBUG 25750 --- []-nio2-thread-2] o.e.j.i.t.sshd.JGitClientSession : sendInitialServiceRequest(JGitClientSession[git#github.com/140.82.121.3:22]) Send SSH_MSG_SERVICE_REQUEST for ssh-userauth
2021-10-23 18:04:50.997 DEBUG 25750 --- []-nio2-thread-2] o.e.j.i.t.sshd.JGitClientSession : encode(JGitClientSession[git#github.com/140.82.121.3:22]) packet #3 sending command=5[SSH_MSG_SERVICE_REQUEST] len=17
2021-10-23 18:04:50.997 DEBUG 25750 --- []-nio2-thread-2] o.a.sshd.common.io.nio2.Nio2Session : writeBuffer(Nio2Session[local=/192.168.178.37:52145, remote=github.com/140.82.121.3:22]) writing 84 bytes
2021-10-23 18:04:50.997 DEBUG 25750 --- []-nio2-thread-2] o.e.j.i.t.sshd.JGitClientSession : encode(JGitClientSession[git#github.com/140.82.121.3:22]) packet #4 sending command=50[SSH_MSG_USERAUTH_REQUEST] len=34
2021-10-23 18:04:50.997 DEBUG 25750 --- []-nio2-thread-2] o.a.sshd.common.io.nio2.Nio2Session : writeBuffer(Nio2Session[local=/192.168.178.37:52145, remote=github.com/140.82.121.3:22]) writing 100 bytes
2021-10-23 18:04:50.997 DEBUG 25750 --- []-nio2-thread-2] o.e.j.i.t.sshd.JGitClientSession : handleNewKeys(JGitClientSession[git#github.com/140.82.121.3:22]) sent 1 pending packets
2021-10-23 18:04:51.099 DEBUG 25750 --- []-nio2-thread-3] s.c.k.e.DefaultClientKexExtensionHandler : handleServerSignatureAlgorithms(JGitClientSession[git#github.com/140.82.121.3:22]): [ssh-ed25519-cert-v01#openssh.com, ecdsa-sha2-nistp521-cert-v01#openssh.com, ecdsa-sha2-nistp384-cert-v01#openssh.com, ecdsa-sha2-nistp256-cert-v01#openssh.com, sk-ssh-ed25519-cert-v01#openssh.com, sk-ecdsa-sha2-nistp256-cert-v01#openssh.com, rsa-sha2-512-cert-v01#openssh.com, rsa-sha2-256-cert-v01#openssh.com, ssh-rsa-cert-v01#openssh.com, ssh-dss-cert-v01#openssh.com, sk-ssh-ed25519#openssh.com, sk-ecdsa-sha2-nistp256#openssh.com, ssh-ed25519, ecdsa-sha2-nistp521, ecdsa-sha2-nistp384, ecdsa-sha2-nistp256, rsa-sha2-512, rsa-sha2-256, ssh-rsa, ssh-dss]
2021-10-23 18:04:51.099 DEBUG 25750 --- []-nio2-thread-3] s.c.k.e.DefaultClientKexExtensionHandler : handleServerSignatureAlgorithms(JGitClientSession[git#github.com/140.82.121.3:22]): PubkeyAcceptedAlgorithms before: [ecdsa-sha2-nistp256-cert-v01#openssh.com, ecdsa-sha2-nistp384-cert-v01#openssh.com, ecdsa-sha2-nistp521-cert-v01#openssh.com, ssh-ed25519-cert-v01#openssh.com, rsa-sha2-512-cert-v01#openssh.com, rsa-sha2-256-cert-v01#openssh.com, ssh-rsa-cert-v01#openssh.com, ecdsa-sha2-nistp256, ecdsa-sha2-nistp384, ecdsa-sha2-nistp521, ssh-ed25519, sk-ecdsa-sha2-nistp256#openssh.com, sk-ssh-ed25519#openssh.com, rsa-sha2-512, rsa-sha2-256, ssh-rsa, ssh-dss-cert-v01#openssh.com, ssh-dss]
2021-10-23 18:04:51.099 DEBUG 25750 --- []-nio2-thread-3] s.c.k.e.DefaultClientKexExtensionHandler : handleServerSignatureAlgorithms(JGitClientSession[git#github.com/140.82.121.3:22]): PubkeyAcceptedAlgorithms after: [ecdsa-sha2-nistp256-cert-v01#openssh.com, ecdsa-sha2-nistp384-cert-v01#openssh.com, ecdsa-sha2-nistp521-cert-v01#openssh.com, ssh-ed25519-cert-v01#openssh.com, rsa-sha2-512-cert-v01#openssh.com, rsa-sha2-256-cert-v01#openssh.com, ssh-rsa-cert-v01#openssh.com, ecdsa-sha2-nistp256, ecdsa-sha2-nistp384, ecdsa-sha2-nistp521, ssh-ed25519, sk-ecdsa-sha2-nistp256#openssh.com, sk-ssh-ed25519#openssh.com, rsa-sha2-512, rsa-sha2-256, ssh-rsa, ssh-dss-cert-v01#openssh.com, ssh-dss]
2021-10-23 18:04:51.118 DEBUG 25750 --- []-nio2-thread-4] o.e.j.i.t.sshd.JGitClientSession : handleServiceAccept(JGitClientSession[git#github.com/140.82.121.3:22]) SSH_MSG_SERVICE_ACCEPT service=ssh-userauth
2021-10-23 18:04:51.157 DEBUG 25750 --- []-nio2-thread-5] o.a.s.c.session.ClientUserAuthService : processUserAuth(JGitClientSession[git#github.com/140.82.121.3:22]) Received SSH_MSG_USERAUTH_FAILURE - partial=false, methods=publickey
2021-10-23 18:04:51.157 DEBUG 25750 --- []-nio2-thread-5] o.a.s.c.session.ClientUserAuthService : tryNext(JGitClientSession[git#github.com/140.82.121.3:22]) starting authentication mechanisms: client=[publickey], server=[publickey]
2021-10-23 18:04:51.157 DEBUG 25750 --- []-nio2-thread-5] o.a.s.c.session.ClientUserAuthService : tryNext(JGitClientSession[git#github.com/140.82.121.3:22]) attempting method=publickey
2021-10-23 18:04:51.158 DEBUG 25750 --- []-nio2-thread-5] o.e.j.i.t.s.JGitPublicKeyAuthentication : sendAuthDataRequest(JGitClientSession[git#github.com/140.82.121.3:22])[ssh-connection] send SSH_MSG_USERAUTH_REQUEST request publickey type=rsa-sha2-512 - fingerprint=SHA256:wtyx9w1r6/ELRZtxJAwXjHmnIyeeQenDhh94q4HJgxE
2021-10-23 18:04:51.158 DEBUG 25750 --- []-nio2-thread-5] o.e.j.i.t.sshd.JGitClientSession : encode(JGitClientSession[git#github.com/140.82.121.3:22]) packet #5 sending command=50[SSH_MSG_USERAUTH_REQUEST] len=211
2021-10-23 18:04:51.158 DEBUG 25750 --- []-nio2-thread-5] o.a.sshd.common.io.nio2.Nio2Session : writeBuffer(Nio2Session[local=/192.168.178.37:52145, remote=github.com/140.82.121.3:22]) writing 276 bytes
2021-10-23 18:04:51.158 DEBUG 25750 --- []-nio2-thread-5] o.a.s.c.session.ClientUserAuthService : tryNext(JGitClientSession[git#github.com/140.82.121.3:22]) successfully processed initial buffer by method=publickey
2021-10-23 18:04:51.282 DEBUG 25750 --- []-nio2-thread-6] o.a.s.c.session.ClientUserAuthService : processUserAuth(JGitClientSession[git#github.com/140.82.121.3:22]) Received SSH_MSG_USERAUTH_FAILURE - partial=false, methods=publickey
2021-10-23 18:04:51.282 DEBUG 25750 --- []-nio2-thread-6] o.e.j.i.t.s.JGitPublicKeyAuthentication : sendAuthDataRequest(JGitClientSession[git#github.com/140.82.121.3:22])[ssh-connection] server rejected publickey authentication with known signature algorithm rsa-sha2-512
2021-10-23 18:04:51.282 DEBUG 25750 --- []-nio2-thread-6] o.e.j.i.t.s.JGitPublicKeyAuthentication : resolveAttemptedPublicKeyIdentity(JGitClientSession[git#github.com/140.82.121.3:22])[ssh-connection] no more keys to send
2021-10-23 18:04:51.282 DEBUG 25750 --- []-nio2-thread-6] o.a.s.c.session.ClientUserAuthService : tryNext(JGitClientSession[git#github.com/140.82.121.3:22]) no initial request sent by method=publickey
2021-10-23 18:04:51.282 DEBUG 25750 --- []-nio2-thread-6] o.e.j.i.t.s.JGitPublicKeyAuthentication : destroy(JGitClientSession[git#github.com/140.82.121.3:22])[ssh-connection]
2021-10-23 18:04:51.282 DEBUG 25750 --- []-nio2-thread-6] o.a.s.c.session.ClientUserAuthService : tryNext(JGitClientSession[git#github.com/140.82.121.3:22]) exhausted all methods - client=[publickey], server=[publickey]
2021-10-23 18:04:52.465 DEBUG 25750 --- [nio-9000-exec-5] o.e.j.i.t.sshd.JGitClientSession : close(JGitClientSession[git#github.com/140.82.121.3:22]) Closing immediately
2021-10-23 18:04:52.466 DEBUG 25750 --- [nio-9000-exec-5] o.e.j.i.t.sshd.JGitClientSession : signalAuthFailure(JGitClientSession[git#github.com/140.82.121.3:22]) type=SshException, signalled=false, first=false: Session is being closed
2021-10-23 18:04:52.466 DEBUG 25750 --- [nio-9000-exec-5] o.a.s.c.s.h.SessionTimeoutListener : sessionClosed(JGitClientSession[git#github.com/140.82.121.3:22]) un-tracked
2021-10-23 18:04:52.466 DEBUG 25750 --- [nio-9000-exec-5] o.a.s.c.session.ClientUserAuthService : close(org.apache.sshd.client.session.ClientUserAuthService#cd3cad2) Closing immediately
2021-10-23 18:04:52.466 DEBUG 25750 --- [nio-9000-exec-5] o.a.s.c.session.ClientUserAuthService : close(org.apache.sshd.client.session.ClientUserAuthService#cd3cad2)[Immediately] closed
2021-10-23 18:04:52.466 DEBUG 25750 --- [nio-9000-exec-5] o.a.s.c.session.ClientConnectionService : close(ClientConnectionService[JGitClientSession[git#github.com/140.82.121.3:22]]) Closing immediately
2021-10-23 18:04:52.466 DEBUG 25750 --- [nio-9000-exec-5] o.a.s.c.session.ClientConnectionService : stopHeartBeat(JGitClientSession[git#github.com/140.82.121.3:22]) no heartbeat to stop
2021-10-23 18:04:52.466 DEBUG 25750 --- [nio-9000-exec-5] o.a.s.c.session.ClientConnectionService : close(ClientConnectionService[JGitClientSession[git#github.com/140.82.121.3:22]])[Immediately] closed
2021-10-23 18:04:52.466 DEBUG 25750 --- [nio-9000-exec-5] o.a.sshd.common.io.nio2.Nio2Session : close(Nio2Session[local=/192.168.178.37:52145, remote=github.com/140.82.121.3:22]) Closing immediately
2021-10-23 18:04:52.466 DEBUG 25750 --- [nio-9000-exec-5] o.a.sshd.common.io.nio2.Nio2Session : doCloseImmediately(Nio2Session[local=/192.168.178.37:52145, remote=github.com/140.82.121.3:22]) closing socket=sun.nio.ch.UnixAsynchronousSocketChannelImpl[connected local=/192.168.178.37:52145 remote=github.com/140.82.121.3:22]
2021-10-23 18:04:52.466 DEBUG 25750 --- [nio-9000-exec-5] o.a.sshd.common.io.nio2.Nio2Session : doCloseImmediately(Nio2Session[local=/192.168.178.37:52145, remote=github.com/140.82.121.3:22]) socket=sun.nio.ch.UnixAsynchronousSocketChannelImpl[closed] closed
2021-10-23 18:04:52.466 DEBUG 25750 --- [nio-9000-exec-5] o.a.sshd.common.io.nio2.Nio2Connector : unmapSession(id=104): Nio2Session[local=/192.168.178.37:52145, remote=github.com/140.82.121.3:22]
2021-10-23 18:04:52.466 DEBUG 25750 --- [nio-9000-exec-5] o.e.j.i.t.sshd.JGitClientSession : close(JGitClientSession[git#github.com/140.82.121.3:22])[Immediately] state already Immediate
2021-10-23 18:04:52.466 DEBUG 25750 --- [nio-9000-exec-5] o.a.sshd.common.io.nio2.Nio2Session : close(Nio2Session[local=/192.168.178.37:52145, remote=github.com/140.82.121.3:22])[Immediately] closed
2021-10-23 18:04:52.466 DEBUG 25750 --- [nio-9000-exec-5] o.a.s.c.u.closeable.SequentialCloseable : doClose(org.apache.sshd.common.util.closeable.SequentialCloseable$1#5b272ea5) signal close complete immediately=true
2021-10-23 18:04:52.466 DEBUG 25750 --- [nio-9000-exec-5] o.e.j.i.t.sshd.JGitClientSession : close(JGitClientSession[git#github.com/140.82.121.3:22])[Immediately] closed
2021-10-23 18:04:52.466 DEBUG 25750 --- []-nio2-thread-7] o.a.sshd.common.io.nio2.Nio2Session : handleReadCycleFailure(Nio2Session[local=/192.168.178.37:52145, remote=github.com/140.82.121.3:22]) AsynchronousCloseException after 1183659791 nanos at read cycle=7: null
2021-10-23 18:04:52.466 DEBUG 25750 --- [nio-9000-exec-5] o.e.j.i.transport.sshd.JGitSshClient : close(JGitSshClient[97d9893]) Closing immediately
2021-10-23 18:04:52.466 DEBUG 25750 --- [nio-9000-exec-5] o.a.sshd.common.io.nio2.Nio2Connector : close(org.apache.sshd.common.io.nio2.Nio2Connector#7cd2a6a6) Closing immediately
2021-10-23 18:04:52.466 DEBUG 25750 --- [nio-9000-exec-5] o.a.sshd.common.io.nio2.Nio2Connector : close(org.apache.sshd.common.io.nio2.Nio2Connector#7cd2a6a6)[Immediately] closed
2021-10-23 18:04:52.467 DEBUG 25750 --- [nio-9000-exec-5] o.a.s.c.u.closeable.SequentialCloseable : doClose(org.apache.sshd.common.util.closeable.SequentialCloseable$1#7cced81d) signal close complete immediately=true
2021-10-23 18:04:52.467 DEBUG 25750 --- [nio-9000-exec-5] o.e.j.i.transport.sshd.JGitSshClient : close(JGitSshClient[97d9893])[Immediately] closed
2021-10-23 18:04:52.468 DEBUG 25750 --- [nio-9000-exec-5] o.s.web.servlet.DispatcherServlet : Failed to complete request: com.mlaide.webserver.service.git.GitDiffException: org.eclipse.jgit.api.errors.TransportException: git#github.com:MLAide/python-client.git: Cannot log in at github.com:22
2021-10-23 18:04:52.469 ERROR 25750 --- [nio-9000-exec-5] o.a.c.c.C.[.[.[/].[dispatcherServlet] : Servlet.service() for servlet [dispatcherServlet] in context with path [] threw exception [Request processing failed; nested exception is com.mlaide.webserver.service.git.GitDiffException: org.eclipse.jgit.api.errors.TransportException: git#github.com:MLAide/python-client.git: Cannot log in at github.com:22] with root cause
org.apache.sshd.common.SshException: No more authentication methods available
at org.apache.sshd.client.session.ClientUserAuthService.tryNext(ClientUserAuthService.java:353) ~[sshd-osgi-2.7.0.jar:2.7.0]
at org.apache.sshd.client.session.ClientUserAuthService.processUserAuth(ClientUserAuthService.java:288) ~[sshd-osgi-2.7.0.jar:2.7.0]
at org.apache.sshd.client.session.ClientUserAuthService.process(ClientUserAuthService.java:225) ~[sshd-osgi-2.7.0.jar:2.7.0]
at org.apache.sshd.common.session.helpers.AbstractSession.doHandleMessage(AbstractSession.java:503) ~[sshd-osgi-2.7.0.jar:2.7.0]
at org.apache.sshd.common.session.helpers.AbstractSession.handleMessage(AbstractSession.java:429) ~[sshd-osgi-2.7.0.jar:2.7.0]
at org.apache.sshd.common.session.helpers.AbstractSession.decode(AbstractSession.java:1466) ~[sshd-osgi-2.7.0.jar:2.7.0]
at org.apache.sshd.common.session.helpers.AbstractSession.messageReceived(AbstractSession.java:389) ~[sshd-osgi-2.7.0.jar:2.7.0]
at org.eclipse.jgit.internal.transport.sshd.JGitClientSession.messageReceived(JGitClientSession.java:198) ~[org.eclipse.jgit.ssh.apache-5.13.0.202109080827-r.jar:5.13.0.202109080827-r]
at org.apache.sshd.common.session.helpers.AbstractSessionIoHandler.messageReceived(AbstractSessionIoHandler.java:64) ~[sshd-osgi-2.7.0.jar:2.7.0]
at org.apache.sshd.common.io.nio2.Nio2Session.handleReadCycleCompletion(Nio2Session.java:359) ~[sshd-osgi-2.7.0.jar:2.7.0]
at org.apache.sshd.common.io.nio2.Nio2Session$1.onCompleted(Nio2Session.java:336) ~[sshd-osgi-2.7.0.jar:2.7.0]
at org.apache.sshd.common.io.nio2.Nio2Session$1.onCompleted(Nio2Session.java:333) ~[sshd-osgi-2.7.0.jar:2.7.0]
at org.apache.sshd.common.io.nio2.Nio2CompletionHandler.lambda$completed$0(Nio2CompletionHandler.java:38) ~[sshd-osgi-2.7.0.jar:2.7.0]
at java.base/java.security.AccessController.doPrivileged(AccessController.java:318) ~[na:na]
at org.apache.sshd.common.io.nio2.Nio2CompletionHandler.completed(Nio2CompletionHandler.java:37) ~[sshd-osgi-2.7.0.jar:2.7.0]
at java.base/sun.nio.ch.Invoker.invokeUnchecked(Invoker.java:129) ~[na:na]
at java.base/sun.nio.ch.Invoker$2.run(Invoker.java:221) ~[na:na]
at java.base/sun.nio.ch.AsynchronousChannelGroupImpl$1.run(AsynchronousChannelGroupImpl.java:113) ~[na:na]
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[na:na]
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[na:na]
at java.base/java.lang.Thread.run(Thread.java:833) ~[na:na]
I used the following library versions:
<dependency>
<groupId>org.eclipse.jgit</groupId>
<artifactId>org.eclipse.jgit</artifactId>
<version>5.13.0.202109080827-r</version>
</dependency>
<dependency>
<groupId>org.eclipse.jgit</groupId>
<artifactId>org.eclipse.jgit.ssh.apache</artifactId>
<version>5.13.0.202109080827-r</version>
</dependency>
Related
I'm using spring-kafka '2.2.7.RELEASE' to create a batch consumer and I'm trying to understand how the consumer rebalancing works when my record processing time exceeds max.poll.interval.ms.
Here is my configurations.
public Map<String, Object> myBatchConsumerConfigs() {
Map<String, Object> configs = = new HashMap<>();
configs.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest");
configs.put(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, "false");
configs.put(ConsumerConfig.MAX_POLL_INTERVAL_MS_CONFIG, "300000");
sapphireKafkaConsumerConfig.setSpecificAvroReader("true");
}
and here are my factory settings
#Bean
public <K,V> ConcurrentKafkaListenerContainerFactory<String, Object> kafkaListenerContainerFactory() {
ConcurrentKafkaListenerContainerFactory<String, Object> factory = new ConcurrentKafkaListenerContainerFactory<>();
factory.setConsumerFactory(new DefaultKafkaConsumerFactory<>(myBatchConsumerConfigs()));
factory.getContainerProperties().setMissingTopicsFatal(false);
factory.getContainerProperties().setAckMode(AckMode.BATCH);
factory.setErrorHandler(myCustomKafkaSeekToCurrentErrorHandler);
factory.setRecoveryCallback(myCustomKafkaRecoveryCallback);
factory.setStatefulRetry(true);
factory.setBatchListener(true);
factory.setBatchErrorHandler(myBatchConsumerSeekToCurrentErrorHandler);
factory.getContainerProperties().setConsumerRebalanceListener(myBatchConsumerAwareRebalanceListener);
factory.setRecoveryCallback(context -> {
logger.logInfo("In recovery call back for KES Batch Consumer", this.getClass());
myBatchConsumerDeadLetterRecoverer.accept((ConsumerRecord<?, ?>) context.getAttribute("record"), (Exception) context.getLastThrowable());
return null;
});
return factory;
}
I've added custom consumer listener as shown here.
#Component
public class MyBatchConsumerAwareRebalanceListener implements ConsumerAwareRebalanceListener {
#Override
public void onPartitionsRevokedBeforeCommit(Consumer<?, ?> consumer, Collection<org.apache.kafka.common.TopicPartition> partitions) {
partitions.forEach(
topicPartition -> {
System.out.println(" onPartitionsRevokedBeforeCommit - topic "+ topicPartition.topic() +" partition - "+topicPartition.partition());
}
);
//springLogger.logInfo(" onPartitionsRevokedBeforeCommit", getClass());
}
#Override
public void onPartitionsRevokedAfterCommit(Consumer<?, ?> consumer, Collection<org.apache.kafka.common.TopicPartition> partitions) {
partitions.forEach(
topicPartition -> {
System.out.println(" onPartitionsRevokedAfterCommit - topic "+ topicPartition.topic() +" partition - "+topicPartition.partition());
}
);
//springLogger.logInfo(" onPartitionsRevokedAfterCommit", getClass());
}
#Override
public void onPartitionsAssigned(Consumer<?, ?> consumer, Collection<org.apache.kafka.common.TopicPartition> partitions) {
partitions.forEach(
topicPartition -> {
System.out.println(" onPartitionsAssigned - topic - "+ topicPartition.topic() +" partition - "+topicPartition.partition());
}
);
//springLogger.logInfo(" onPartitionsAssigned", getClass());
}
}
Here is my consumer where I've added a delay of 400 sec delay/sleep which is greater than the max.poll.interval.ms ( 300 ms)
#KafkaListener(groupId = "TestBatchConsumers", topics = TEST_KES_BATCH_CONSUMER_TOPIC, containerFactory = "myBatchConsumerContainerFactory")
public void consumeRecords(List<ConsumerRecord<String, Organization>> consumerRecords) {
long startTime = System.currentTimeMillis();
System.out.println("Processing started at "+startTime);
consumerRecords.forEach(consumerRecord -> {
System.out.println(
"Received consumerRecord on topic" + consumerRecord.topic()+" , partition "+consumerRecord.partition()
+ ", at offset " + consumerRecord.offset() + ", with key " + consumerRecord.key() );
try {
Thread.sleep(400000);
} catch (InterruptedException e) {
e.printStackTrace();
}
});
System.out.println("Processing completed at "+ System.currentTimeMillis());
long processingTimeInSec = (System.currentTimeMillis() - startTime)/1000 ;
System.out.println(processingTimeInSec);
}
Now i was expecting the consumer group to have rebalance because the processing time is more than max.poll.interval.ms but I didn't see any such behavior. Am i missing something here?
Please suggest.
It behaves as expected for me:
#SpringBootApplication
public class So67520619Application {
public static void main(String[] args) {
SpringApplication.run(So67520619Application.class, args);
}
#Bean
public ApplicationRunner runner(KafkaTemplate<String, String> template) {
return args -> template.send("so67520619", "foo");
}
#Bean
public NewTopic topic() {
return TopicBuilder.name("so67520619").partitions(1).replicas(1).build();
}
}
#Component
class Listener implements ConsumerAwareRebalanceListener {
private static final Logger LOG = LoggerFactory.getLogger(Listener.class);
#KafkaListener(id = "so67520619", topics = "so67520619")
public void listen(List<String> in) throws InterruptedException {
LOG.info(in.toString());
Thread.sleep(12000);
}
#Override
public void onPartitionsAssigned(Consumer<?, ?> consumer, Collection<TopicPartition> partitions) {
LOG.info("Assigned: " + partitions);
}
#Override
public void onPartitionsRevokedAfterCommit(Consumer<?, ?> consumer, Collection<TopicPartition> partitions) {
LOG.info("Revoked: " + partitions);
}
}
spring.kafka.consumer.properties.max.poll.interval.ms=10000
spring.kafka.consumer.auto-offset-reset=earliest
spring.kafka.listener.type=batch
2021-05-13 11:13:16.339 INFO 20954 --- [o67520619-0-C-1] com.example.demo.Listener : Assigned: [so67520619-0]
2021-05-13 11:13:16.358 INFO 20954 --- [o67520619-0-C-1] com.example.demo.Listener : [foo, foo, foo, foo]
2021-05-13 11:13:26.416 INFO 20954 --- [ad | so67520619] o.a.k.c.c.internals.AbstractCoordinator : [Consumer clientId=consumer-so67520619-1, groupId=so67520619] Member consumer-so67520619-1-c9a440bf-9076-4575-813d-3efb0054f5f7 sending LeaveGroup request to coordinator localhost:9092 (id: 2147483647 rack: null) due to consumer poll timeout has expired. This means the time between subsequent calls to poll() was longer than the configured max.poll.interval.ms, which typically implies that the poll loop is spending too much time processing messages. You can address this either by increasing max.poll.interval.ms or by reducing the maximum size of batches returned in poll() with max.poll.records.
2021-05-13 11:13:28.365 INFO 20954 --- [o67520619-0-C-1] o.a.k.c.c.internals.ConsumerCoordinator : [Consumer clientId=consumer-so67520619-1, groupId=so67520619] Failing OffsetCommit request since the consumer is not part of an active group
2021-05-13 11:13:28.370 ERROR 20954 --- [o67520619-0-C-1] essageListenerContainer$ListenerConsumer : Consumer exception
org.springframework.kafka.KafkaException: Seek to current after exception; nested exception is org.apache.kafka.clients.consumer.CommitFailedException: Offset commit cannot be completed since the consumer is not part of an active group for auto partition assignment; it is likely that the consumer was kicked out of the group.
at org.springframework.kafka.listener.SeekToCurrentBatchErrorHandler.handle(SeekToCurrentBatchErrorHandler.java:72) ~[spring-kafka-2.6.7.jar:2.6.7]
at org.springframework.kafka.listener.RecoveringBatchErrorHandler.handle(RecoveringBatchErrorHandler.java:124) ~[spring-kafka-2.6.7.jar:2.6.7]
at org.springframework.kafka.listener.KafkaMessageListenerContainer$ListenerConsumer.handleConsumerException(KafkaMessageListenerContainer.java:1431) ~[spring-kafka-2.6.7.jar:2.6.7]
at org.springframework.kafka.listener.KafkaMessageListenerContainer$ListenerConsumer.run(KafkaMessageListenerContainer.java:1124) ~[spring-kafka-2.6.7.jar:2.6.7]
at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515) ~[na:na]
at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[na:na]
at java.base/java.lang.Thread.run(Thread.java:832) ~[na:na]
Caused by: org.apache.kafka.clients.consumer.CommitFailedException: Offset commit cannot be completed since the consumer is not part of an active group for auto partition assignment; it is likely that the consumer was kicked out of the group.
at org.apache.kafka.clients.consumer.internals.ConsumerCoordinator.sendOffsetCommitRequest(ConsumerCoordinator.java:1134) ~[kafka-clients-2.6.0.jar:na]
at org.apache.kafka.clients.consumer.internals.ConsumerCoordinator.commitOffsetsSync(ConsumerCoordinator.java:999) ~[kafka-clients-2.6.0.jar:na]
at org.apache.kafka.clients.consumer.KafkaConsumer.commitSync(KafkaConsumer.java:1504) ~[kafka-clients-2.6.0.jar:na]
at org.springframework.kafka.listener.KafkaMessageListenerContainer$ListenerConsumer.doCommitSync(KafkaMessageListenerContainer.java:2396) ~[spring-kafka-2.6.7.jar:2.6.7]
at org.springframework.kafka.listener.KafkaMessageListenerContainer$ListenerConsumer.commitSync(KafkaMessageListenerContainer.java:2391) ~[spring-kafka-2.6.7.jar:2.6.7]
at org.springframework.kafka.listener.KafkaMessageListenerContainer$ListenerConsumer.commitIfNecessary(KafkaMessageListenerContainer.java:2377) ~[spring-kafka-2.6.7.jar:2.6.7]
at org.springframework.kafka.listener.KafkaMessageListenerContainer$ListenerConsumer.processCommits(KafkaMessageListenerContainer.java:2191) ~[spring-kafka-2.6.7.jar:2.6.7]
at org.springframework.kafka.listener.KafkaMessageListenerContainer$ListenerConsumer.pollAndInvoke(KafkaMessageListenerContainer.java:1149) ~[spring-kafka-2.6.7.jar:2.6.7]
at org.springframework.kafka.listener.KafkaMessageListenerContainer$ListenerConsumer.run(KafkaMessageListenerContainer.java:1075) ~[spring-kafka-2.6.7.jar:2.6.7]
... 3 common frames omitted
2021-05-13 11:13:28.371 INFO 20954 --- [o67520619-0-C-1] o.a.k.c.c.internals.ConsumerCoordinator : [Consumer clientId=consumer-so67520619-1, groupId=so67520619] Giving away all assigned partitions as lost since generation has been reset,indicating that consumer is no longer part of the group
2021-05-13 11:13:28.371 INFO 20954 --- [o67520619-0-C-1] o.a.k.c.c.internals.ConsumerCoordinator : [Consumer clientId=consumer-so67520619-1, groupId=so67520619] Lost previously assigned partitions so67520619-0
2021-05-13 11:13:28.371 INFO 20954 --- [o67520619-0-C-1] com.example.demo.Listener : Revoked: [so67520619-0]
2021-05-13 11:13:28.372 INFO 20954 --- [o67520619-0-C-1] o.a.k.c.c.internals.AbstractCoordinator : [Consumer clientId=consumer-so67520619-1, groupId=so67520619] (Re-)joining group
2021-05-13 11:13:28.376 INFO 20954 --- [o67520619-0-C-1] o.a.k.c.c.internals.AbstractCoordinator : [Consumer clientId=consumer-so67520619-1, groupId=so67520619] Join group failed with org.apache.kafka.common.errors.MemberIdRequiredException: The group member needs to have a valid member id before actually entering a consumer group.
2021-05-13 11:13:28.376 INFO 20954 --- [o67520619-0-C-1] o.a.k.c.c.internals.AbstractCoordinator : [Consumer clientId=consumer-so67520619-1, groupId=so67520619] (Re-)joining group
2021-05-13 11:13:28.485 INFO 20954 --- [o67520619-0-C-1] o.a.k.c.c.internals.ConsumerCoordinator : [Consumer clientId=consumer-so67520619-1, groupId=so67520619] Finished assignment for group at generation 3: {consumer-so67520619-1-15ce3150-1aa3-4b43-a892-fbd54e8ed919=Assignment(partitions=[so67520619-0])}
2021-05-13 11:13:28.486 INFO 20954 --- [o67520619-0-C-1] o.a.k.c.c.internals.AbstractCoordinator : [Consumer clientId=consumer-so67520619-1, groupId=so67520619] Successfully joined group with generation 3
2021-05-13 11:13:28.487 INFO 20954 --- [o67520619-0-C-1] o.a.k.c.c.internals.ConsumerCoordinator : [Consumer clientId=consumer-so67520619-1, groupId=so67520619] Notifying assignor about the new Assignment(partitions=[so67520619-0])
2021-05-13 11:13:28.487 INFO 20954 --- [o67520619-0-C-1] o.a.k.c.c.internals.ConsumerCoordinator : [Consumer clientId=consumer-so67520619-1, groupId=so67520619] Adding newly assigned partitions: so67520619-0
2021-05-13 11:13:28.488 INFO 20954 --- [o67520619-0-C-1] o.a.k.c.c.internals.ConsumerCoordinator : [Consumer clientId=consumer-so67520619-1, groupId=so67520619] Found no committed offset for partition so67520619-0
2021-05-13 11:13:28.489 INFO 20954 --- [o67520619-0-C-1] o.a.k.c.c.internals.SubscriptionState : [Consumer clientId=consumer-so67520619-1, groupId=so67520619] Resetting offset for partition so67520619-0 to offset 0.
2021-05-13 11:13:28.490 INFO 20954 --- [o67520619-0-C-1] com.example.demo.Listener : Assigned: [so67520619-0]
2021-05-13 11:13:28.494 INFO 20954 --- [o67520619-0-C-1] com.example.demo.Listener : [foo, foo, foo, foo]
EDIT
And with 60/65 seconds, it still works for me...
2021-05-13 17:24:28.111 INFO 37063 --- [o67520619-0-C-1] com.example.demo.Listener : Assigned: [so67520619-0]
2021-05-13 17:24:28.130 INFO 37063 --- [o67520619-0-C-1] com.example.demo.Listener : [foo, foo, foo, foo, foo, foo]
2021-05-13 17:25:28.147 INFO 37063 --- [ad | so67520619] o.a.k.c.c.internals.AbstractCoordinator : [Consumer clientId=consumer-so67520619-1, groupId=so67520619] Member consumer-so67520619-1-269ac261-3838-4925-a9a7-fd0687db3522 sending LeaveGroup request to coordinator localhost:9092 (id: 2147483647 rack: null) due to consumer poll timeout has expired. This means the time between subsequent calls to poll() was longer than the configured max.poll.interval.ms, which typically implies that the poll loop is spending too much time processing messages. You can address this either by increasing max.poll.interval.ms or by reducing the maximum size of batches returned in poll() with max.poll.records.
2021-05-13 17:25:33.135 INFO 37063 --- [o67520619-0-C-1] o.a.k.c.c.internals.ConsumerCoordinator : [Consumer clientId=consumer-so67520619-1, groupId=so67520619] Failing OffsetCommit request since the consumer is not part of an active group
2021-05-13 17:25:33.141 ERROR 37063 --- [o67520619-0-C-1] essageListenerContainer$ListenerConsumer : Consumer exception
...
2021-05-13 17:25:33.141 INFO 37063 --- [o67520619-0-C-1] o.a.k.c.c.internals.ConsumerCoordinator : [Consumer clientId=consumer-so67520619-1, groupId=so67520619] Giving away all assigned partitions as lost since generation has been reset,indicating that consumer is no longer part of the group
2021-05-13 17:25:33.141 INFO 37063 --- [o67520619-0-C-1] o.a.k.c.c.internals.ConsumerCoordinator : [Consumer clientId=consumer-so67520619-1, groupId=so67520619] Lost previously assigned partitions so67520619-0
2021-05-13 17:25:33.141 INFO 37063 --- [o67520619-0-C-1] com.example.demo.Listener : Revoked: [so67520619-0]
2021-05-13 17:25:33.142 INFO 37063 --- [o67520619-0-C-1] o.a.k.c.c.internals.AbstractCoordinator : [Consumer clientId=consumer-so67520619-1, groupId=so67520619] (Re-)joining group
2021-05-13 17:25:33.145 INFO 37063 --- [o67520619-0-C-1] o.a.k.c.c.internals.AbstractCoordinator : [Consumer clientId=consumer-so67520619-1, groupId=so67520619] Join group failed with org.apache.kafka.common.errors.MemberIdRequiredException: The group member needs to have a valid member id before actually entering a consumer group.
2021-05-13 17:25:33.145 INFO 37063 --- [o67520619-0-C-1] o.a.k.c.c.internals.AbstractCoordinator : [Consumer clientId=consumer-so67520619-1, groupId=so67520619] (Re-)joining group
2021-05-13 17:25:33.250 INFO 37063 --- [o67520619-0-C-1] o.a.k.c.c.internals.ConsumerCoordinator : [Consumer clientId=consumer-so67520619-1, groupId=so67520619] Finished assignment for group at generation 9: {consumer-so67520619-1-bd22a252-64f2-4be3-a6eb-8371b8f95ff2=Assignment(partitions=[so67520619-0])}
2021-05-13 17:25:33.254 INFO 37063 --- [o67520619-0-C-1] o.a.k.c.c.internals.AbstractCoordinator : [Consumer clientId=consumer-so67520619-1, groupId=so67520619] Successfully joined group with generation 9
2021-05-13 17:25:33.254 INFO 37063 --- [o67520619-0-C-1] o.a.k.c.c.internals.ConsumerCoordinator : [Consumer clientId=consumer-so67520619-1, groupId=so67520619] Notifying assignor about the new Assignment(partitions=[so67520619-0])
2021-05-13 17:25:33.255 INFO 37063 --- [o67520619-0-C-1] o.a.k.c.c.internals.ConsumerCoordinator : [Consumer clientId=consumer-so67520619-1, groupId=so67520619] Adding newly assigned partitions: so67520619-0
2021-05-13 17:25:33.256 INFO 37063 --- [o67520619-0-C-1] o.a.k.c.c.internals.ConsumerCoordinator : [Consumer clientId=consumer-so67520619-1, groupId=so67520619] Found no committed offset for partition so67520619-0
2021-05-13 17:25:33.258 INFO 37063 --- [o67520619-0-C-1] o.a.k.c.c.internals.SubscriptionState : [Consumer clientId=consumer-so67520619-1, groupId=so67520619] Resetting offset for partition so67520619-0 to offset 0.
2021-05-13 17:25:33.258 INFO 37063 --- [o67520619-0-C-1] com.example.demo.Listener : Assigned: [so67520619-0]
2021-05-13 17:25:33.261 INFO 37063 --- [o67520619-0-C-1] com.example.demo.Listener : [foo, foo, foo, foo, foo, foo]
Kafka complains after 60 seconds and the commit fails 5 seconds later.
I use spring security with roles to protect current pages or actions with their own addresses of my application. With access denied page i can catch page of error 403 (access denied) when user tries to
get not allowed page, but when i protect current action, like this:
<form th:action="'/blog/'+${el.id}+'/remove'" method="post">
<button class = "btn btn-warning" type="submit">Delete</button>
</form>
With current controller's method:
#PostMapping("/blog/{id}/remove")
public String blogPostRemove(#PathVariable(value = "id") long postId, Model model) {
Post post = postRepository.findById(postId).orElseThrow();
postRepository.delete(post);
return "redirect:/blog";
}
When user with not allowed role tries to act he will get page with error 405:
There was an unexpected error (type=Method Not Allowed, status=405).
Request method 'POST' not supported. User with allowed role doesn't have any problem. There is a WebSecurityConfig code:
#EnableWebSecurity
public class WebSecurityConfig extends WebSecurityConfigurerAdapter {
#Override
protected void configure(HttpSecurity http) throws Exception {
http.authorizeRequests()
.antMatchers("/unpublished","/blog/*/remove","/blog/*/comments/*/remove")
.hasAuthority("ADMIN")
.antMatchers("/blog/*/edit","/blog/*/comment","/blog/add").authenticated()
.antMatchers("/","/registration","/login","/blog/*").permitAll()
.and().formLogin()
.loginPage("/login")
.and().exceptionHandling()
.accessDeniedPage("/access-denied");
http.csrf().disable();
}
}
How can i change this page with my /access-denied?
Logs:
2020-07-13 11:26:58.094 DEBUG 42636 --- [nio-8080-exec-6] o.s.web.servlet.DispatcherServlet : GET "/blog/28", parameters={}
2020-07-13 11:26:58.096 DEBUG 42636 --- [nio-8080-exec-6] s.w.s.m.m.a.RequestMappingHandlerMapping : Mapped to com.example.springExample.controllers.BlogController#blogDetails(long, Model)
2020-07-13 11:26:58.364 DEBUG 42636 --- [nio-8080-exec-6] o.s.w.s.v.ContentNegotiatingViewResolver : Selected 'text/html' given [text/html, application/xhtml+xml, image/webp, image/apng, application/xml;q=0.9, application/signed-exchange;v=b3;q=0.9, *//*;q=0.8]
2020-07-13 11:26:58.402 DEBUG 42636 --- [nio-8080-exec-6] o.s.web.servlet.DispatcherServlet : Completed 200 OK
2020-07-13 11:27:04.814 DEBUG 42636 --- [nio-8080-exec-9] o.s.web.servlet.DispatcherServlet : "FORWARD" dispatch for POST "/access-denied", parameters={}
2020-07-13 11:27:04.816 WARN 42636 --- [nio-8080-exec-9] .w.s.m.s.DefaultHandlerExceptionResolver : Resolved [org.springframework.web.HttpRequestMethodNotSupportedException: Request method 'POST' not supported]
2020-07-13 11:27:04.817 DEBUG 42636 --- [nio-8080-exec-9] o.s.web.servlet.DispatcherServlet : Exiting from "FORWARD" dispatch, status 405
2020-07-13 11:27:04.818 DEBUG 42636 --- [nio-8080-exec-9] o.s.web.servlet.DispatcherServlet : "ERROR" dispatch for POST "/error", parameters={}
2020-07-13 11:27:04.818 DEBUG 42636 --- [nio-8080-exec-9] s.w.s.m.m.a.RequestMappingHandlerMapping : Mapped to org.springframework.boot.autoconfigure.web.servlet.error.BasicErrorController#errorHtml(HttpServletRequest, HttpServletResponse)
2020-07-13 11:27:04.831 DEBUG 42636 --- [nio-8080-exec-9] o.s.w.s.v.ContentNegotiatingViewResolver : Selected 'text/html' given [text/html, text/html;q=0.8]
2020-07-13 11:27:04.838 DEBUG 42636 --- [nio-8080-exec-9] o.s.web.servlet.DispatcherServlet : Exiting from "ERROR" dispatch, status 405
I located the problem - when user tries to act, he will send post method from form. When application catches an access-denied error, it will try to get /access-denied page, but due to action type, it will be post request. Solution is to add PostMapping function of /access-denied page to the current controller:
#GetMapping("/access-denied")
public String blockAccess(Model model) {
return "access-denied";
}
#PostMapping("/access-denied")
public String actionAccess(Model model) {
return "access-denied";
}
When i use a #ControllerAdvice with the #ExceptionHandler annotation then none of the normal exception handling works anymore. All results in a HTML page with the title "HTTP Status 500 – Internal Server Error" and nothing is logged in the console.
I've created a simple #ControllerAdvice as below, when this didn't work as expected i started trying with the basePackages and extending from ResponseEntityExceptionHandler.
#ControllerAdvice(basePackages = "nl.xxxx.events")
public class EventExceptionHandler extends ResponseEntityExceptionHandler {
private static final Logger logger = LoggerFactory.getLogger(EventExceptionHandler.class);
#ResponseStatus(HttpStatus.NOT_FOUND)
#ExceptionHandler(EventNotFoundException.class)
public void handlePersonNotFound(RuntimeException ex) {
logger.error("error", ex);
}
}
In the service i throw the exception using:
public Event findById(Long id) {
return this.eventRepository.findById(id).orElseThrow(() -> new EventNotFoundException(id));
}
I've tried to compare the flow of code with and without #ControllerAdvice but there are numerous things that are different. For example without the #ControllerAdvice the value attribute of the HttpEntityMethodProcessor is set with the an object describing the exception. But with the #ControllerAdvice this attribute is always null.
I expected the code above to not interfere with the normal exception handling.
As a example i'd like to use the "could not initialize proxy - no Session" error because this is easy for me to reproduce.
Before i added the #ControllerAdvice spring gave the following json result:
{"timestamp":"2019-06-02T19:17:14.223+0000","status":500,"error":"Internal Server Error","message":"Could not write JSON: failed to lazily initialize a collection of role: (truncated)....","path":"/events/find"}
and with debug logging:
2019-06-02 21:18:29.723 DEBUG 1588 --- [nio-8080-exec-2] o.s.web.servlet.DispatcherServlet : GET "/events/find?page=1&size=10", parameters={masked}
2019-06-02 21:18:29.724 DEBUG 1588 --- [nio-8080-exec-2] s.w.s.m.m.a.RequestMappingHandlerMapping : Mapped to public org.springframework.data.domain.Page<nl.xxxx.events.Event> nl.xxxx.events.EventController.findPersons(java.lang.String,java.lang.Integer,java.lang.Integer)
2019-06-02 21:18:29.728 DEBUG 1588 --- [nio-8080-exec-2] m.m.a.RequestResponseBodyMethodProcessor : Using 'application/json', given [application/json, text/plain, */*] and supported [application/json, application/*+json, application/json, application/*+json]
2019-06-02 21:18:29.728 DEBUG 1588 --- [nio-8080-exec-2] m.m.a.RequestResponseBodyMethodProcessor : Writing [Page 1 of 19 containing nl.xxxx.events.Event instances]
2019-06-02 21:18:29.729 WARN 1588 --- [nio-8080-exec-2] .w.s.m.s.DefaultHandlerExceptionResolver : Resolved [org.springframework.http.converter.HttpMessageNotWritableException: Could not write JSON: failed to lazily initialize a collection of role: nl.xxxx.events.Event.schedule, could not initialize proxy - no Session; nested exception is com.fasterxml.jackson.databind.JsonMappingException: failed to lazily initialize a collection of role: nl.xxxx.events.Event.schedule, could not initialize proxy - no Session (through reference chain: org.springframework.data.domain.PageImpl["content"]->java.util.Collections$UnmodifiableRandomAccessList[0]->nl.xxxx.events.Event["schedule"])]
2019-06-02 21:18:29.729 DEBUG 1588 --- [nio-8080-exec-2] o.s.web.servlet.DispatcherServlet : Completed 500 INTERNAL_SERVER_ERROR
2019-06-02 21:18:29.729 DEBUG 1588 --- [nio-8080-exec-2] o.s.web.servlet.DispatcherServlet : "ERROR" dispatch for GET "/error?page=1&size=10", parameters={masked}
2019-06-02 21:18:29.730 DEBUG 1588 --- [nio-8080-exec-2] s.w.s.m.m.a.RequestMappingHandlerMapping : Mapped to public org.springframework.http.ResponseEntity<java.util.Map<java.lang.String, java.lang.Object>> org.springframework.boot.autoconfigure.web.servlet.error.BasicErrorController.error(javax.servlet.http.HttpServletRequest)
2019-06-02 21:18:29.731 DEBUG 1588 --- [nio-8080-exec-2] o.s.w.s.m.m.a.HttpEntityMethodProcessor : Using 'application/json', given [application/json, text/plain, */*] and supported [application/json, application/*+json, application/json, application/*+json]
2019-06-02 21:18:29.731 DEBUG 1588 --- [nio-8080-exec-2] o.s.w.s.m.m.a.HttpEntityMethodProcessor : Writing [{timestamp=Sun Jun 02 21:18:29 CEST 2019, status=500, error=Internal Server Error, message=Could not (truncated)...]
2019-06-02 21:18:29.732 DEBUG 1588 --- [nio-8080-exec-2] o.s.web.servlet.DispatcherServlet : Exiting from "ERROR" dispatch, status 500
But now it returns:
<!doctype html><html lang="en"><head><title>HTTP Status 500 – Internal Server Error</title><style type="text/css">h1 {font-family:Tahoma,Arial,sans-serif;color:white;background-color:#525D76;font-size:22px;} h2 {font-family:Tahoma,Arial,sans-serif;color:white;background-color:#525D76;font-size:16px;} h3 {font-family:Tahoma,Arial,sans-serif;color:white;background-color:#525D76;font-size:14px;} body {font-family:Tahoma,Arial,sans-serif;color:black;background-color:white;} b {font-family:Tahoma,Arial,sans-serif;color:white;background-color:#525D76;} p {font-family:Tahoma,Arial,sans-serif;background:white;color:black;font-size:12px;} a {color:black;} a.name {color:black;} .line {height:1px;background-color:#525D76;border:none;}</style></head><body><h1>HTTP Status 500 – Internal Server Error</h1></body></html>
And with debug logging:
2019-06-02 21:20:33.682 DEBUG 22308 --- [nio-8080-exec-1] o.s.web.servlet.DispatcherServlet : GET "/events/find?page=1&size=10", parameters={masked}
2019-06-02 21:20:33.688 DEBUG 22308 --- [nio-8080-exec-1] s.w.s.m.m.a.RequestMappingHandlerMapping : Mapped to public org.springframework.data.domain.Page<nl.xxxx.events.Event> nl.xxxx.events.EventController.findPersons(java.lang.String,java.lang.Integer,java.lang.Integer)
2019-06-02 21:20:33.760 INFO 22308 --- [nio-8080-exec-1] o.h.h.i.QueryTranslatorFactoryInitiator : HHH000397: Using ASTQueryTranslatorFactory
2019-06-02 21:20:33.900 DEBUG 22308 --- [nio-8080-exec-1] m.m.a.RequestResponseBodyMethodProcessor : Using 'application/json', given [application/json, text/plain, */*] and supported [application/json, application/*+json, application/json, application/*+json]
2019-06-02 21:20:33.901 DEBUG 22308 --- [nio-8080-exec-1] m.m.a.RequestResponseBodyMethodProcessor : Writing [Page 1 of 19 containing nl.xxxx.events.Event instances]
2019-06-02 21:20:33.923 DEBUG 22308 --- [nio-8080-exec-1] .m.m.a.ExceptionHandlerExceptionResolver : Using #ExceptionHandler public final org.springframework.http.ResponseEntity<java.lang.Object> org.springframework.web.servlet.mvc.method.annotation.ResponseEntityExceptionHandler.handleException(java.lang.Exception,org.springframework.web.context.request.WebRequest) throws java.lang.Exception
2019-06-02 21:20:33.925 DEBUG 22308 --- [nio-8080-exec-1] o.s.w.s.m.m.a.HttpEntityMethodProcessor : No match for [application/json, text/plain, */*], supported: []
2019-06-02 21:20:33.926 DEBUG 22308 --- [nio-8080-exec-1] .m.m.a.ExceptionHandlerExceptionResolver : Resolved [org.springframework.http.converter.HttpMessageNotWritableException: Could not write JSON: failed to lazily initialize a collection of role: nl.xxxx.events.Event.schedule, could not initialize proxy - no Session; nested exception is com.fasterxml.jackson.databind.JsonMappingException: failed to lazily initialize a collection of role: nl.xxxx.events.Event.schedule, could not initialize proxy - no Session (through reference chain: org.springframework.data.domain.PageImpl["content"]->java.util.Collections$UnmodifiableRandomAccessList[0]->nl.xxxx.events.Event["schedule"])]
2019-06-02 21:20:33.926 DEBUG 22308 --- [nio-8080-exec-1] o.s.web.servlet.DispatcherServlet : Completed 500 INTERNAL_SERVER_ERROR
So, at some point it just says No match for [application/json, text/plain, */*], supported: [] and wont return a nice JSON exception anymore.
Any ideas at what i'm doing wrong? initially is just want EventNotFoundException to result in a 404 instead of a 500. Propably in the future i'd like to add content aswell.
The problem is that you try to handle RuntimeException which is too general. public void handlePersonNotFound(RuntimeException ex) You also don't need to extend from ResponseEntityExceptionHandler.
If you write an ExceptionHandler that should handle a specific exception, the handleException method needs to accept that specific exception as parameter. Also you would probably want some sort of repsonse.
#ControllerAdvice
public class EventExceptionHandler {
private static final Logger logger = LoggerFactory.getLogger(getClass());
#ResponseBody
#ResponseStatus(HttpStatus.NOT_FOUND)
#ExceptionHandler(EventNotFoundException.class)
public String handlePersonNotFound(EventNotFoundException ex) {
logger.error("error", ex);
return ex.getMessage();
}
}
This will give a basic reponse body with the exception message and error code.
The code below enables 2nd level retry and IErrorHandler.
The problem is that IErrorHandler is never invoked.
var activator = new BuiltinHandlerActivator();
activator.Register((bus, mc) => new Handler(mc, bus));
Configure.With(activator)
.Transport(t => t.UseAzureServiceBus(Consts.ServiceBusConnectionString, Consts.Subscriber1))
.Options(o =>
{
o.SimpleRetryStrategy(maxDeliveryAttempts: 2,
secondLevelRetriesEnabled: true,
errorQueueAddress: "poison");
o.Decorate<IErrorHandler>(c => new MyErrorHandler(c.Get<IErrorHandler>()));
}).Start();
Handler
class Handler : IHandleMessages<string>,
IHandleMessages<IFailed<object>>
{
readonly IBus _bus;
readonly IMessageContext _messageContext;
public Handler(IMessageContext messageContext, IBus bus)
{
_messageContext = messageContext;
_bus = bus;
}
public async Task Handle(string message)
{
Console.WriteLine("Handle(string message): {0}", message);
throw new Exception("Handle(string message)");
}
public async Task Handle(IFailed<Object> message)
{
Console.WriteLine("Handle(IFailed<Object> message): {0}", message);
await _bus.Advanced.TransportMessage.Defer(TimeSpan.FromSeconds(2));
}
}
Specifically, below is the sequence of running 1st and 2nd level retry:
Calling sequences based on maxDeliveryAttempts is 2:
1 calling Handle(string message),
throw exception within Handle(string message)
2 calling Handle(string message)
throw exception within Handle(string message)
3 calling Handle(IFailed message)
4 calling Handle(IFailed message)
5 calling Handle(string message),
throw exception within Handle(string message)
6 calling Handle(IFailed message)
Repeat 5 and 6 forever
Question A:
Why is IErrorHandler not invoked?
How to invoke IErrorHandler after calling Handle(IFailed message) 2 times based on maxDeliveryAttempts of 2.
That is:
Calling sequences based on maxDeliveryAttempts is 2:
1 calling Handle(string message),
throw exception within Handle(string message)
2 calling Handle(string message)
throw exception within Handle(string message)
3 calling Handle(IFailed message)
4 calling Handle(IFailed message)
5 calling IErrorHandler
6 Move to next message
Update
The same issue occurs on IHandleMessages<IFailed<string>>.
Update 2
Error below:
Severity Code Description Project File Line Suppression State
Error Unable to locate repository containing directory
'C:\ReBus\Rebus-master\Rebus'. Rebus C:\Users\username\.nuget\packages\microsoft.build.tasks.git\1.0.0-beta-63127-02\build\Microsoft.Build.Tasks.Git.targets 20
Clicking the error above shows this in VS
<Microsoft.Build.Tasks.Git.LocateRepository Directory="$(MSBuildProjectDirectory)" >
<Output TaskParameter="Id" PropertyName="_GitRepositoryRoot" />
</Microsoft.Build.Tasks.Git.LocateRepository>
Update 3
Result below of running this https://github.com/rebus-org/Rebus/blob/master/Rebus.Tests/Bugs/VerifyThisParticularThingAboutSecondLevelRetries.cs
[INF] Rebus.Threading.TaskParallelLibrary.TplAsyncTask (Worker#3): Starting periodic task "CleanupTrackedErrors" with interval 00:00:10
[INF] Rebus.Threading.TaskParallelLibrary.TplAsyncTask (Worker#3): Starting periodic task "DueMessagesSender" with interval 00:00:01
[INF] Rebus.Bus.RebusBus (Worker#3): Bus "Rebus 1" setting number of workers to 1
[DBG] Rebus.Bus.RebusBus (Worker#3): Adding worker "Rebus 1 worker 1"
[INF] Rebus.Bus.RebusBus (Worker#3): Bus "Rebus 1" started
[DBG] Rebus.Workers.ThreadPoolBased.ThreadPoolWorker (Rebus 1 worker 1): Starting (threadpool-based) worker "Rebus 1 worker 1"
[DBG] Rebus.Pipeline.Send.SendOutgoingMessageStep (Worker#3): Sending "HEJ MED DIG" -> "whatever"
Handle(string message): HEJ MED DIG
[WRN] Rebus.Retry.ErrorTracking.InMemErrorTracker (Rebus 1 worker 1): Unhandled exception 1 while handling message with ID "667c8cfd-5ce5-46a6-9394-32d1386ef7de"
System.Exception: Handle(string message)
at Rebus.Tests.Bugs.VerifyThisParticularThingAboutSecondLevelRetries.Handler.<Handle>d__4.MoveNext() in C:\ReBus\Rebus.Tests\Bugs\VerifyThisParticularThingAboutSecondLevelRetries.cs:line 81
--- End of stack trace from previous location where exception was thrown ---
at System.Runtime.ExceptionServices.ExceptionDispatchInfo.Throw()
at System.Runtime.CompilerServices.TaskAwaiter.HandleNonSuccessAndDebuggerNotification(Task task)
at System.Runtime.CompilerServices.TaskAwaiter.GetResult()
at Rebus.Pipeline.Receive.HandlerInvoker`1.<Invoke>d__14.MoveNext() in C:\ReBus\Rebus\Pipeline\Receive\HandlerInvoker.cs:line 154
--- End of stack trace from previous location where exception was thrown ---
at System.Runtime.ExceptionServices.ExceptionDispatchInfo.Throw()
at System.Runtime.CompilerServices.TaskAwaiter.HandleNonSuccessAndDebuggerNotification(Task task)
at System.Runtime.CompilerServices.TaskAwaiter.GetResult()
at Rebus.Pipeline.Receive.DispatchIncomingMessageStep.<Process>d__3.MoveNext() in C:\ReBus\Rebus\Pipeline\Receive\DispatchIncomingMessageStep.cs:line 66
--- End of stack trace from previous location where exception was thrown ---
at System.Runtime.ExceptionServices.ExceptionDispatchInfo.Throw()
at System.Runtime.CompilerServices.TaskAwaiter.HandleNonSuccessAndDebuggerNotification(Task task)
at System.Runtime.CompilerServices.TaskAwaiter.GetResult()
at Rebus.Sagas.LoadSagaDataStep.<Process>d__6.MoveNext() in C:\ReBus\Rebus\Sagas\LoadSagaDataStep.cs:line 66
--- End of stack trace from previous location where exception was thrown ---
at System.Runtime.ExceptionServices.ExceptionDispatchInfo.Throw()
at System.Runtime.CompilerServices.TaskAwaiter.HandleNonSuccessAndDebuggerNotification(Task task)
at System.Runtime.CompilerServices.TaskAwaiter.GetResult()
at Rebus.Pipeline.Receive.ActivateHandlersStep.<Process>d__3.MoveNext() in C:\ReBus\Rebus\Pipeline\Receive\ActivateHandlersStep.cs:line 47
--- End of stack trace from previous location where exception was thrown ---
at System.Runtime.ExceptionServices.ExceptionDispatchInfo.Throw()
at System.Runtime.CompilerServices.TaskAwaiter.HandleNonSuccessAndDebuggerNotification(Task task)
at System.Runtime.CompilerServices.TaskAwaiter.GetResult()
at Rebus.Pipeline.Receive.HandleRoutingSlipsStep.<Process>d__3.MoveNext() in C:\ReBus\Rebus\Pipeline\Receive\HandleRoutingSlipsStep.cs:line 40
--- End of stack trace from previous location where exception was thrown ---
at System.Runtime.ExceptionServices.ExceptionDispatchInfo.Throw()
at System.Runtime.CompilerServices.TaskAwaiter.HandleNonSuccessAndDebuggerNotification(Task task)
at System.Runtime.CompilerServices.TaskAwaiter.GetResult()
at Rebus.Retry.Simple.FailedMessageWrapperStep.<Process>d__2.MoveNext() in C:\ReBus\Rebus\Retry\Simple\FailedMessageWrapperStep.cs:line 42
--- End of stack trace from previous location where exception was thrown ---
at System.Runtime.ExceptionServices.ExceptionDispatchInfo.Throw()
at System.Runtime.CompilerServices.TaskAwaiter.HandleNonSuccessAndDebuggerNotification(Task task)
at System.Runtime.CompilerServices.TaskAwaiter.GetResult()
at Rebus.Pipeline.Receive.DeserializeIncomingMessageStep.<Process>d__2.MoveNext() in C:\ReBus\Rebus\Pipeline\Receive\DeserializeIncomingMessageStep.cs:line 34
--- End of stack trace from previous location where exception was thrown ---
at System.Runtime.ExceptionServices.ExceptionDispatchInfo.Throw()
at System.Runtime.CompilerServices.TaskAwaiter.HandleNonSuccessAndDebuggerNotification(Task task)
at System.Runtime.CompilerServices.TaskAwaiter.GetResult()
at Rebus.DataBus.ClaimCheck.HydrateIncomingMessageStep.<Process>d__2.MoveNext() in C:\ReBus\Rebus\DataBus\ClaimCheck\HydrateIncomingMessageStep.cs:line 51
--- End of stack trace from previous location where exception was thrown ---
at System.Runtime.ExceptionServices.ExceptionDispatchInfo.Throw()
at System.Runtime.CompilerServices.TaskAwaiter.HandleNonSuccessAndDebuggerNotification(Task task)
at System.Runtime.CompilerServices.TaskAwaiter.GetResult()
at Rebus.Pipeline.Receive.HandleDeferredMessagesStep.<Process>d__12.MoveNext() in C:\ReBus\Rebus\Pipeline\Receive\HandleDeferredMessagesStep.cs:line 121
--- End of stack trace from previous location where exception was thrown ---
at System.Runtime.ExceptionServices.ExceptionDispatchInfo.Throw()
at System.Runtime.CompilerServices.TaskAwaiter.HandleNonSuccessAndDebuggerNotification(Task task)
at System.Runtime.CompilerServices.TaskAwaiter.GetResult()
at Rebus.Retry.FailFast.FailFastStep.<Process>d__3.MoveNext() in C:\ReBus\Rebus\Retry\FailFast\FailFastStep.cs:line 51
--- End of stack trace from previous location where exception was thrown ---
at System.Runtime.ExceptionServices.ExceptionDispatchInfo.Throw()
at System.Runtime.CompilerServices.TaskAwaiter.HandleNonSuccessAndDebuggerNotification(Task task)
at System.Runtime.CompilerServices.TaskAwaiter.GetResult()
at Rebus.Retry.Simple.SimpleRetryStrategyStep.<DispatchWithTrackerIdentifier>d__10.MoveNext() in C:\ReBus\Rebus\Retry\Simple\SimpleRetryStrategyStep.cs:line 118
Handle(string message): HEJ MED DIG
[WRN] Rebus.Retry.ErrorTracking.InMemErrorTracker (Rebus 1 worker 1): Unhandled exception 2 while handling message with ID "667c8cfd-5ce5-46a6-9394-32d1386ef7de"
System.Exception: Handle(string message)
at Rebus.Tests.Bugs.VerifyThisParticularThingAboutSecondLevelRetries.Handler.<Handle>d__4.MoveNext() in C:\ReBus\Rebus.Tests\Bugs\VerifyThisParticularThingAboutSecondLevelRetries.cs:line 81
--- End of stack trace from previous location where exception was thrown ---
at System.Runtime.ExceptionServices.ExceptionDispatchInfo.Throw()
at System.Runtime.CompilerServices.TaskAwaiter.HandleNonSuccessAndDebuggerNotification(Task task)
at System.Runtime.CompilerServices.TaskAwaiter.GetResult()
at Rebus.Pipeline.Receive.HandlerInvoker`1.<Invoke>d__14.MoveNext() in C:\ReBus\Rebus\Pipeline\Receive\HandlerInvoker.cs:line 154
--- End of stack trace from previous location where exception was thrown ---
at System.Runtime.ExceptionServices.ExceptionDispatchInfo.Throw()
at System.Runtime.CompilerServices.TaskAwaiter.HandleNonSuccessAndDebuggerNotification(Task task)
at System.Runtime.CompilerServices.TaskAwaiter.GetResult()
at Rebus.Pipeline.Receive.DispatchIncomingMessageStep.<Process>d__3.MoveNext() in C:\ReBus\Rebus\Pipeline\Receive\DispatchIncomingMessageStep.cs:line 66
--- End of stack trace from previous location where exception was thrown ---
at System.Runtime.ExceptionServices.ExceptionDispatchInfo.Throw()
at System.Runtime.CompilerServices.TaskAwaiter.HandleNonSuccessAndDebuggerNotification(Task task)
at System.Runtime.CompilerServices.TaskAwaiter.GetResult()
at Rebus.Sagas.LoadSagaDataStep.<Process>d__6.MoveNext() in C:\ReBus\Rebus\Sagas\LoadSagaDataStep.cs:line 66
--- End of stack trace from previous location where exception was thrown ---
at System.Runtime.ExceptionServices.ExceptionDispatchInfo.Throw()
at System.Runtime.CompilerServices.TaskAwaiter.HandleNonSuccessAndDebuggerNotification(Task task)
at System.Runtime.CompilerServices.TaskAwaiter.GetResult()
at Rebus.Pipeline.Receive.ActivateHandlersStep.<Process>d__3.MoveNext() in C:\ReBus\Rebus\Pipeline\Receive\ActivateHandlersStep.cs:line 47
--- End of stack trace from previous location where exception was thrown ---
at System.Runtime.ExceptionServices.ExceptionDispatchInfo.Throw()
at System.Runtime.CompilerServices.TaskAwaiter.HandleNonSuccessAndDebuggerNotification(Task task)
at System.Runtime.CompilerServices.TaskAwaiter.GetResult()
at Rebus.Pipeline.Receive.HandleRoutingSlipsStep.<Process>d__3.MoveNext() in C:\ReBus\Rebus\Pipeline\Receive\HandleRoutingSlipsStep.cs:line 40
--- End of stack trace from previous location where exception was thrown ---
at System.Runtime.ExceptionServices.ExceptionDispatchInfo.Throw()
at System.Runtime.CompilerServices.TaskAwaiter.HandleNonSuccessAndDebuggerNotification(Task task)
at System.Runtime.CompilerServices.TaskAwaiter.GetResult()
at Rebus.Retry.Simple.FailedMessageWrapperStep.<Process>d__2.MoveNext() in C:\ReBus\Rebus\Retry\Simple\FailedMessageWrapperStep.cs:line 42
--- End of stack trace from previous location where exception was thrown ---
at System.Runtime.ExceptionServices.ExceptionDispatchInfo.Throw()
at System.Runtime.CompilerServices.TaskAwaiter.HandleNonSuccessAndDebuggerNotification(Task task)
at System.Runtime.CompilerServices.TaskAwaiter.GetResult()
at Rebus.Pipeline.Receive.DeserializeIncomingMessageStep.<Process>d__2.MoveNext() in C:\ReBus\Rebus\Pipeline\Receive\DeserializeIncomingMessageStep.cs:line 34
--- End of stack trace from previous location where exception was thrown ---
at System.Runtime.ExceptionServices.ExceptionDispatchInfo.Throw()
at System.Runtime.CompilerServices.TaskAwaiter.HandleNonSuccessAndDebuggerNotification(Task task)
at System.Runtime.CompilerServices.TaskAwaiter.GetResult()
at Rebus.DataBus.ClaimCheck.HydrateIncomingMessageStep.<Process>d__2.MoveNext() in C:\ReBus\Rebus\DataBus\ClaimCheck\HydrateIncomingMessageStep.cs:line 51
--- End of stack trace from previous location where exception was thrown ---
at System.Runtime.ExceptionServices.ExceptionDispatchInfo.Throw()
at System.Runtime.CompilerServices.TaskAwaiter.HandleNonSuccessAndDebuggerNotification(Task task)
at System.Runtime.CompilerServices.TaskAwaiter.GetResult()
at Rebus.Pipeline.Receive.HandleDeferredMessagesStep.<Process>d__12.MoveNext() in C:\ReBus\Rebus\Pipeline\Receive\HandleDeferredMessagesStep.cs:line 121
--- End of stack trace from previous location where exception was thrown ---
at System.Runtime.ExceptionServices.ExceptionDispatchInfo.Throw()
at System.Runtime.CompilerServices.TaskAwaiter.HandleNonSuccessAndDebuggerNotification(Task task)
at System.Runtime.CompilerServices.TaskAwaiter.GetResult()
at Rebus.Retry.FailFast.FailFastStep.<Process>d__3.MoveNext() in C:\ReBus\Rebus\Retry\FailFast\FailFastStep.cs:line 51
--- End of stack trace from previous location where exception was thrown ---
at System.Runtime.ExceptionServices.ExceptionDispatchInfo.Throw()
at System.Runtime.CompilerServices.TaskAwaiter.HandleNonSuccessAndDebuggerNotification(Task task)
at System.Runtime.CompilerServices.TaskAwaiter.GetResult()
at Rebus.Retry.Simple.SimpleRetryStrategyStep.<DispatchWithTrackerIdentifier>d__10.MoveNext() in C:\ReBus\Rebus\Retry\Simple\SimpleRetryStrategyStep.cs:line 118
Handle(IFailed<Object> message): FAILED: HEJ MED DIG
[DBG] Rebus.Pipeline.Receive.DispatchIncomingMessageStep (Rebus 1 worker 1): Dispatching "System.String, mscorlib" "667c8cfd-5ce5-46a6-9394-32d1386ef7de" to 1 handlers took 4 ms
Disposing Rebus.Activation.BuiltinHandlerActivator
[DBG] Rebus.Pipeline.Receive.HandleDeferredMessagesStep (Rebus 1 worker 1): Deferring message "String/667c8cfd-5ce5-46a6-9394-32d1386ef7de" until 2019-06-04T10:36:51.6554132+01:00
[INF] Rebus.Bus.RebusBus (Worker#3): Bus "Rebus 1" setting number of workers to 0
[DBG] Rebus.Workers.ThreadPoolBased.ThreadPoolWorker (Rebus 1 worker 1): Worker "Rebus 1 worker 1" stopped
[DBG] Rebus.Bus.RebusBus (Worker#3): Removing worker "Rebus 1 worker 1"
[INF] Rebus.Threading.TaskParallelLibrary.TplAsyncTask (Worker#3): Stopping periodic task "DueMessagesSender"
[INF] Rebus.Threading.TaskParallelLibrary.TplAsyncTask (Worker#3): Stopping periodic task "CleanupTrackedErrors"
[INF] Rebus.Bus.RebusBus (Worker#3): Bus "Rebus 1" stopped
Disposing System.Threading.ManualResetEvent
What do the logs say?
To help you get on your way, I just created this little test case with a verification that it does on fact work as expected: VerifyThisParticularThingAboutSecondLevelRetries.cs
If you run the test, it should log a bunch of stuff (namely a couple of stack traces from the delivery attempts), and then the message should be dispatched as an Failed<string>, which is compatible with the handler for IFailed<object>.
With Spring Integration, is it possible to synchronize files (including subfolder's files) recursively using ftp inbound channel adapter?
No; but you can fetch a complete remote directory tree with recursive MGET with the outbound gateway...
#SpringBootApplication
#IntegrationComponentScan
public class So42324318Application {
public static void main(String[] args) {
ConfigurableApplicationContext context = SpringApplication.run(So42324318Application.class, args);
List<File> files = context.getBean(Gateway.class).fetchFiles("foo/*");
System.out.println(files);
context.close();
}
#MessagingGateway(defaultRequestChannel = "fetchRecursive")
public interface Gateway {
public List<File> fetchFiles(String remoteDir);
}
#Bean
#ServiceActivator(inputChannel = "fetchRecursive")
public FtpOutboundGateway gateway() {
// Create a recursive MGET gateway that gets the remote directory from the payload
FtpOutboundGateway gateway = new FtpOutboundGateway(sessionFactory(), "mget", "payload");
gateway.setOptions("-R");
gateway.setLocalDirectoryExpression(new SpelExpressionParser().parseExpression("#remoteDirectory"));
return gateway;
}
#Bean
public SessionFactory<FTPFile> sessionFactory() {
return new CachingSessionFactory<>(ftpSF());
}
private SessionFactory<FTPFile> ftpSF() {
DefaultFtpSessionFactory sf = new DefaultFtpSessionFactory();
sf.setHost("10.0.0.3");
sf.setUsername("ftptest");
sf.setPassword("ftptest");
sf.setClientMode(FTPClient.PASSIVE_LOCAL_DATA_CONNECTION_MODE);
return sf;
}
}
Result:
2017-02-19 09:55:09.351 INFO 61921 --- [ main] o.s.integration.ftp.session.FtpSession : File has been successfully transferred from: foo/bar.tx
2017-02-19 09:55:09.353 INFO 61921 --- [ main] o.s.integration.ftp.session.FtpSession : File has been successfully transferred from: foo/bar.txt
2017-02-19 09:55:09.356 INFO 61921 --- [ main] o.s.integration.ftp.session.FtpSession : File has been successfully transferred from: foo/bar/abc.txt
2017-02-19 09:55:09.358 INFO 61921 --- [ main] o.s.integration.ftp.session.FtpSession : File has been successfully transferred from: foo/baz.txt
2017-02-19 09:55:09.362 INFO 61921 --- [ main] o.s.integration.ftp.session.FtpSession : File has been successfully transferred from: foo/foo/bar/qux.txt
2017-02-19 09:55:09.364 INFO 61921 --- [ main] o.s.integration.ftp.session.FtpSession : File has been successfully transferred from: foo/foo/baz/fiz.txt
[foo/bar.tx, foo/bar.txt, foo/bar/abc.txt, foo/baz.txt, foo/foo/bar/qux.txt, foo/foo/baz/fiz.txt]