Skip to content

Commit 3792ec6

Browse files
rickchengxrickchengx
authored
Merge 89db373 into 30fd43e 
2 parents 30fd43e + 89db373 commit 3792ec6

11 files changed

Lines changed: 288 additions & 39 deletions

File tree

‎docs/docs/en/guide/remote-logging.md‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -33,3 +33,20 @@ remote.logging.oss.bucket.name=<bucket.name>
3333
remote.logging.oss.endpoint=<endpoint>
3434
```
3535

36+
## Writing task logs to [Amazon S3](https://aws.amazon.com/cn/s3/)
37+
38+
Configure `common.properties` as follows:
39+
40+
```properties
41+
# s3 access key id, required if you set remote.logging.target=S3
42+
remote.logging.s3.access.key.id=<access.key.id>
43+
# s3 access key secret, required if you set remote.logging.target=S3
44+
remote.logging.s3.access.key.secret=<access.key.secret>
45+
# s3 bucket name, required if you set remote.logging.target=S3
46+
remote.logging.s3.bucket.name=<bucket.name>
47+
# s3 endpoint, required if you set remote.logging.target=S3
48+
remote.logging.s3.endpoint=<endpoint>
49+
# s3 region, required if you set remote.logging.target=S3
50+
remote.logging.s3.region=<region>
51+
```
52+

‎docs/docs/zh/guide/remote-logging.md‎

Lines changed: 18 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@ Apache DolphinScheduler支持将任务日志传输到远端存储上。当配置
1010
```properties
1111
# 是否开启远程日志存储
1212
remote.logging.enable=true
13-
# 任务日志写入的远端存储，目前仅支持OSS
13+
# 任务日志写入的远端存储，目前支持OSS, S3
1414
remote.logging.target=OSS
1515
# 任务日志在远端存储上的目录
1616
remote.logging.base.dir=logs
@@ -33,3 +33,20 @@ remote.logging.oss.bucket.name=<bucket.name>
3333
remote.logging.oss.endpoint=<endpoint>
3434
```
3535

36+
## 将任务日志写入[Amazon S3](https://aws.amazon.com/cn/s3/)
37+
38+
配置`common.propertis`如下：
39+
40+
```properties
41+
# s3 access key id, required if you set remote.logging.target=S3
42+
remote.logging.s3.access.key.id=<access.key.id>
43+
# s3 access key secret, required if you set remote.logging.target=S3
44+
remote.logging.s3.access.key.secret=<access.key.secret>
45+
# s3 bucket name, required if you set remote.logging.target=S3
46+
remote.logging.s3.bucket.name=<bucket.name>
47+
# s3 endpoint, required if you set remote.logging.target=S3
48+
remote.logging.s3.endpoint=<endpoint>
49+
# s3 region, required if you set remote.logging.target=S3
50+
remote.logging.s3.region=<region>
51+
```
52+

‎dolphinscheduler-common/pom.xml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -88,6 +88,11 @@
8888
<artifactId>aliyun-sdk-oss</artifactId>
8989
</dependency>
9090

91+
<dependency>
92+
<groupId>com.amazonaws</groupId>
93+
<artifactId>aws-java-sdk-s3</artifactId>
94+
</dependency>
95+
9196
<dependency>
9297
<groupId>com.github.oshi</groupId>
9398
<artifactId>oshi-core</artifactId>

‎dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/constants/Constants.java‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -829,4 +829,18 @@ private Constants() {
829829
public static final String REMOTE_LOGGING_OSS_BUCKET_NAME = "remote.logging.oss.bucket.name";
830830

831831
public static final String REMOTE_LOGGING_OSS_ENDPOINT = "remote.logging.oss.endpoint";
832+
833+
/**
834+
* remote logging for S3
835+
*/
836+
837+
public static final String REMOTE_LOGGING_S3_ACCESS_KEY_ID = "remote.logging.s3.access.key.id";
838+
839+
public static final String REMOTE_LOGGING_S3_ACCESS_KEY_SECRET = "remote.logging.s3.access.key.secret";
840+
841+
public static final String REMOTE_LOGGING_S3_BUCKET_NAME = "remote.logging.s3.bucket.name";
842+
843+
public static final String REMOTE_LOGGING_S3_ENDPOINT = "remote.logging.s3.endpoint";
844+
845+
public static final String REMOTE_LOGGING_S3_REGION = "remote.logging.s3.region";
832846
}

‎dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/log/remote/OssRemoteLogHandler.java‎

Lines changed: 15 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -27,8 +27,6 @@
2727
import java.io.Closeable;
2828
import java.io.File;
2929
import java.io.IOException;
30-
import java.nio.file.Path;
31-
import java.nio.file.Paths;
3230

3331
import lombok.extern.slf4j.Slf4j;
3432

@@ -40,13 +38,23 @@
4038
@Slf4j
4139
public class OssRemoteLogHandler implements RemoteLogHandler, Closeable {
4240

43-
private static final int OBJECT_NAME_COUNT = 2;
44-
4541
private OSS ossClient;
4642

4743
private String bucketName;
4844

49-
public OssRemoteLogHandler() {
45+
private static OssRemoteLogHandler instance;
46+
47+
private OssRemoteLogHandler() {
48+
49+
}
50+
51+
public static synchronized OssRemoteLogHandler getInstance() {
52+
if (instance == null) {
53+
instance = new OssRemoteLogHandler();
54+
instance.init();
55+
}
56+
57+
return instance;
5058
}
5159

5260
public void init() {
@@ -61,7 +69,7 @@ public void init() {
6169

6270
@Override
6371
public void sendRemoteLog(String logPath) {
64-
String objectName = getObjectNameFromLogPath(logPath);
72+
String objectName = RemoteLogUtils.getObjectNameFromLogPath(logPath);
6573

6674
try {
6775
log.info("send remote log {} to OSS {}", logPath, objectName);
@@ -74,7 +82,7 @@ public void sendRemoteLog(String logPath) {
7482

7583
@Override
7684
public void getRemoteLog(String logPath) {
77-
String objectName = getObjectNameFromLogPath(logPath);
85+
String objectName = RemoteLogUtils.getObjectNameFromLogPath(logPath);
7886

7987
try {
8088
log.info("get remote log on OSS {} to {}", objectName, logPath);
@@ -91,18 +99,6 @@ public void close() throws IOException {
9199
}
92100
}
93101

94-
private String getObjectNameFromLogPath(String logPath) {
95-
Path path = Paths.get(logPath);
96-
int nameCount = path.getNameCount();
97-
98-
if (nameCount < OBJECT_NAME_COUNT) {
99-
return Paths.get(readOssBaseDir(), logPath).toString();
100-
} else {
101-
return Paths.get(readOssBaseDir(), path.subpath(nameCount - OBJECT_NAME_COUNT, nameCount).toString())
102-
.toString();
103-
}
104-
}
105-
106102
private void checkBucketNameExists(String bucketName) {
107103
if (StringUtils.isBlank(bucketName)) {
108104
throw new IllegalArgumentException(Constants.REMOTE_LOGGING_OSS_BUCKET_NAME + " is empty");
@@ -136,8 +132,4 @@ private String readOssEndpoint() {
136132
private String readOssBucketName() {
137133
return PropertyUtils.getString(Constants.REMOTE_LOGGING_OSS_BUCKET_NAME);
138134
}
139-
140-
private String readOssBaseDir() {
141-
return PropertyUtils.getString(Constants.REMOTE_LOGGING_BASE_DIR);
142-
}
143135
}

‎dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/log/remote/RemoteLogHandlerFactory.java‎

Lines changed: 11 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -21,19 +21,25 @@
2121
import org.apache.dolphinscheduler.common.utils.PropertyUtils;
2222

2323
import lombok.experimental.UtilityClass;
24+
import lombok.extern.slf4j.Slf4j;
2425

2526
@UtilityClass
27+
@Slf4j
2628
public class RemoteLogHandlerFactory {
2729

2830
public RemoteLogHandler getRemoteLogHandler() {
2931
if (!RemoteLogUtils.isRemoteLoggingEnable()) {
3032
return null;
3133
}
32-
if (!"OSS".equals(PropertyUtils.getUpperCaseString(Constants.REMOTE_LOGGING_TARGET))) {
33-
return null;
34+
35+
String target = PropertyUtils.getUpperCaseString(Constants.REMOTE_LOGGING_TARGET);
36+
if ("OSS".equals(target)) {
37+
return OssRemoteLogHandler.getInstance();
38+
} else if ("S3".equals(target)) {
39+
return S3RemoteLogHandler.getInstance();
3440
}
35-
OssRemoteLogHandler ossRemoteLogHandler = new OssRemoteLogHandler();
36-
ossRemoteLogHandler.init();
37-
return ossRemoteLogHandler;
41+
42+
log.error("No suitable remote logging target for {}", target);
43+
return null;
3844
}
3945
}

‎dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/log/remote/RemoteLogService.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -41,7 +41,7 @@ public void asyncSendRemoteLog(String logPath) {
4141
return;
4242
}
4343
remoteLogHandler.sendRemoteLog(logPath);
44-
log.info("Succeed to send log {} to remote target {}", logPath,
44+
log.info("End send log {} to remote target {}", logPath,
4545
PropertyUtils.getString(Constants.REMOTE_LOGGING_TARGET));
4646
}
4747
}

‎dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/log/remote/RemoteLogUtils.java‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,8 @@ public class RemoteLogUtils {
3636

3737
private static RemoteLogService remoteLogService;
3838

39+
private static final int OBJECT_NAME_COUNT = 2;
40+
3941
@Autowired
4042
private RemoteLogService autowiredRemoteLogService;
4143

@@ -76,4 +78,18 @@ private static void mkdirOfLog(String logPath) {
7678
public static boolean isRemoteLoggingEnable() {
7779
return PropertyUtils.getBoolean(Constants.REMOTE_LOGGING_ENABLE, Boolean.FALSE);
7880
}
81+
82+
public static String getObjectNameFromLogPath(String logPath) {
83+
Path path = Paths.get(logPath);
84+
int nameCount = path.getNameCount();
85+
86+
String logBaseDir = PropertyUtils.getString(Constants.REMOTE_LOGGING_BASE_DIR);
87+
88+
if (nameCount < OBJECT_NAME_COUNT) {
89+
return Paths.get(logBaseDir, logPath).toString();
90+
} else {
91+
return Paths.get(logBaseDir, path.subpath(nameCount - OBJECT_NAME_COUNT, nameCount).toString())
92+
.toString();
93+
}
94+
}
7995
}

0 commit comments

Comments
 (0)