修复因RFC3986理解有误导致的UrlPath处理冒号转义问题

This commit is contained in:
Looly 2024-07-04 12:27:15 +08:00
parent 1ef70d5210
commit 7e7575c512
4 changed files with 95 additions and 79 deletions

View File

@ -7,6 +7,7 @@
### 🐣新特性
### 🐞Bug修复
* 【core 】 修复因RFC3986理解有误导致的UrlPath处理冒号转义问题issue#IAAE88@Gitee
-------------------------------------------------------------------------------------------------------------
# 5.8.29(2024-07-03)

View File

@ -151,14 +151,9 @@ public class UrlPath {
final char[] safeChars = encodePercent ? null : new char[]{'%'};
final StringBuilder builder = new StringBuilder();
for (final String segment : segments) {
if(builder.length() == 0){
// 根据https://www.ietf.org/rfc/rfc3986.html#section-3.3定义
// path的第一部分不允许有":"其余部分允许
// 在此处的Path部分特指host之后的部分即不包含第一部分
builder.append(CharUtil.SLASH).append(RFC3986.SEGMENT_NZ_NC.encode(segment, charset, safeChars));
} else {
builder.append(CharUtil.SLASH).append(RFC3986.SEGMENT.encode(segment, charset, safeChars));
}
// https://www.ietf.org/rfc/rfc3986.html#section-3.3
// 此处Path中是允许有`:`之前理解有误应该是相对URI的第一个segment中不允许有`:`
builder.append(CharUtil.SLASH).append(RFC3986.SEGMENT.encode(segment, charset, safeChars));
}
if(withEngTag){

View File

@ -1,7 +1,9 @@
package cn.hutool.core.net;
import cn.hutool.core.date.DateUtil;
import cn.hutool.core.lang.Console;
import cn.hutool.core.net.url.UrlBuilder;
import cn.hutool.core.net.url.UrlPath;
import cn.hutool.core.util.CharsetUtil;
import org.junit.Assert;
import org.junit.Test;
@ -12,34 +14,36 @@ import java.nio.charset.StandardCharsets;
import java.util.HashMap;
import java.util.LinkedHashMap;
import static org.junit.Assert.assertEquals;
public class UrlBuilderTest {
@Test
public void buildTest() {
UrlBuilder builder = UrlBuilder.of();
final String buildUrl = builder.setHost("www.hutool.cn").build();
Assert.assertEquals("http://www.hutool.cn/", buildUrl);
Assert.assertEquals(buildUrl, 80, builder.getPortWithDefault());
assertEquals("http://www.hutool.cn/", buildUrl);
assertEquals(buildUrl, 80, builder.getPortWithDefault());
}
@Test
public void buildWithoutSlashTest() {
// https://github.com/dromara/hutool/issues/2459
String buildUrl = UrlBuilder.of().setScheme("http").setHost("192.168.1.1").setPort(8080).setWithEndTag(false).build();
Assert.assertEquals("http://192.168.1.1:8080", buildUrl);
assertEquals("http://192.168.1.1:8080", buildUrl);
UrlBuilder urlBuilder = UrlBuilder.of();
buildUrl = urlBuilder.setScheme("http").setHost("192.168.1.1").setPort(8080).addQuery("url", "http://192.168.1.1/test/1")
.setWithEndTag(false).build();
Assert.assertEquals("http://192.168.1.1:8080?url=http://192.168.1.1/test/1", buildUrl);
Assert.assertEquals(buildUrl, 8080, urlBuilder.getPortWithDefault());
assertEquals("http://192.168.1.1:8080?url=http://192.168.1.1/test/1", buildUrl);
assertEquals(buildUrl, 8080, urlBuilder.getPortWithDefault());
}
@Test
public void buildTest2() {
// path中的+不做处理
final String buildUrl = UrlBuilder.ofHttp("http://www.hutool.cn/+8618888888888", CharsetUtil.CHARSET_UTF_8).build();
Assert.assertEquals("http://www.hutool.cn/+8618888888888", buildUrl);
assertEquals("http://www.hutool.cn/+8618888888888", buildUrl);
}
@Test
@ -47,7 +51,7 @@ public class UrlBuilderTest {
final String buildUrl = UrlBuilder.of()
.setScheme("https")
.setHost("www.hutool.cn").build();
Assert.assertEquals("https://www.hutool.cn/", buildUrl);
assertEquals("https://www.hutool.cn/", buildUrl);
}
@Test
@ -57,7 +61,7 @@ public class UrlBuilderTest {
.setHost("www.hutool.cn")
.setPort(8080)
.build();
Assert.assertEquals("https://www.hutool.cn:8080/", buildUrl);
assertEquals("https://www.hutool.cn:8080/", buildUrl);
}
@Test
@ -70,7 +74,7 @@ public class UrlBuilderTest {
.addQuery("wd", "test")
.build();
Assert.assertEquals("https://www.hutool.cn/aaa/bbb?ie=UTF-8&wd=test", buildUrl);
assertEquals("https://www.hutool.cn/aaa/bbb?ie=UTF-8&wd=test", buildUrl);
}
@Test
@ -83,7 +87,7 @@ public class UrlBuilderTest {
.addQuery("wd", "测试")
.build();
Assert.assertEquals("https://www.hutool.cn/aaa/bbb?ie=UTF-8&wd=%E6%B5%8B%E8%AF%95", buildUrl);
assertEquals("https://www.hutool.cn/aaa/bbb?ie=UTF-8&wd=%E6%B5%8B%E8%AF%95", buildUrl);
}
@Test
@ -97,7 +101,7 @@ public class UrlBuilderTest {
.addQuery("wd", "测试")
.build();
Assert.assertEquals("https://www.hutool.cn/s?ie=UTF-8&ie=GBK&wd=%E6%B5%8B%E8%AF%95", buildUrl);
assertEquals("https://www.hutool.cn/s?ie=UTF-8&ie=GBK&wd=%E6%B5%8B%E8%AF%95", buildUrl);
}
@Test
@ -106,7 +110,7 @@ public class UrlBuilderTest {
.setScheme("https")
.setHost("www.hutool.cn")
.setFragment("abc").build();
Assert.assertEquals("https://www.hutool.cn/#abc", buildUrl);
assertEquals("https://www.hutool.cn/#abc", buildUrl);
}
@Test
@ -115,7 +119,7 @@ public class UrlBuilderTest {
.setScheme("https")
.setHost("www.hutool.cn")
.setFragment("测试").build();
Assert.assertEquals("https://www.hutool.cn/#%E6%B5%8B%E8%AF%95", buildUrl);
assertEquals("https://www.hutool.cn/#%E6%B5%8B%E8%AF%95", buildUrl);
}
@Test
@ -125,7 +129,7 @@ public class UrlBuilderTest {
.setHost("www.hutool.cn")
.addPath("/s")
.setFragment("测试").build();
Assert.assertEquals("https://www.hutool.cn/s#%E6%B5%8B%E8%AF%95", buildUrl);
assertEquals("https://www.hutool.cn/s#%E6%B5%8B%E8%AF%95", buildUrl);
}
@Test
@ -136,22 +140,22 @@ public class UrlBuilderTest {
.addPath("/s")
.addQuery("wd", "test")
.setFragment("测试").build();
Assert.assertEquals("https://www.hutool.cn/s?wd=test#%E6%B5%8B%E8%AF%95", buildUrl);
assertEquals("https://www.hutool.cn/s?wd=test#%E6%B5%8B%E8%AF%95", buildUrl);
}
@Test
public void ofTest() {
final UrlBuilder builder = UrlBuilder.of("http://www.hutool.cn/aaa/bbb/?a=1&b=2#frag1", CharsetUtil.CHARSET_UTF_8);
Assert.assertEquals("http", builder.getScheme());
Assert.assertEquals("www.hutool.cn", builder.getHost());
assertEquals("http", builder.getScheme());
assertEquals("www.hutool.cn", builder.getHost());
Assert.assertEquals("aaa", builder.getPath().getSegment(0));
Assert.assertEquals("bbb", builder.getPath().getSegment(1));
assertEquals("aaa", builder.getPath().getSegment(0));
assertEquals("bbb", builder.getPath().getSegment(1));
Assert.assertEquals("1", builder.getQuery().get("a"));
Assert.assertEquals("2", builder.getQuery().get("b"));
assertEquals("1", builder.getQuery().get("a"));
assertEquals("2", builder.getQuery().get("b"));
Assert.assertEquals("frag1", builder.getFragment());
assertEquals("frag1", builder.getFragment());
}
@Test
@ -164,47 +168,47 @@ public class UrlBuilderTest {
@Test
public void ofWithChineseTest() {
final UrlBuilder builder = UrlBuilder.ofHttp("www.hutool.cn/aaa/bbb/?a=张三&b=%e6%9d%8e%e5%9b%9b#frag1", CharsetUtil.CHARSET_UTF_8);
Assert.assertEquals("http", builder.getScheme());
Assert.assertEquals("www.hutool.cn", builder.getHost());
assertEquals("http", builder.getScheme());
assertEquals("www.hutool.cn", builder.getHost());
Assert.assertEquals("aaa", builder.getPath().getSegment(0));
Assert.assertEquals("bbb", builder.getPath().getSegment(1));
assertEquals("aaa", builder.getPath().getSegment(0));
assertEquals("bbb", builder.getPath().getSegment(1));
Assert.assertEquals("张三", builder.getQuery().get("a"));
Assert.assertEquals("李四", builder.getQuery().get("b"));
assertEquals("张三", builder.getQuery().get("a"));
assertEquals("李四", builder.getQuery().get("b"));
Assert.assertEquals("frag1", builder.getFragment());
assertEquals("frag1", builder.getFragment());
}
@Test
public void ofWithBlankTest() {
final UrlBuilder builder = UrlBuilder.ofHttp(" www.hutool.cn/aaa/bbb/?a=张三&b=%e6%9d%8e%e5%9b%9b#frag1", CharsetUtil.CHARSET_UTF_8);
Assert.assertEquals("http", builder.getScheme());
Assert.assertEquals("www.hutool.cn", builder.getHost());
assertEquals("http", builder.getScheme());
assertEquals("www.hutool.cn", builder.getHost());
Assert.assertEquals("aaa", builder.getPath().getSegment(0));
Assert.assertEquals("bbb", builder.getPath().getSegment(1));
assertEquals("aaa", builder.getPath().getSegment(0));
assertEquals("bbb", builder.getPath().getSegment(1));
Assert.assertEquals("张三", builder.getQuery().get("a"));
Assert.assertEquals("李四", builder.getQuery().get("b"));
assertEquals("张三", builder.getQuery().get("a"));
assertEquals("李四", builder.getQuery().get("b"));
Assert.assertEquals("frag1", builder.getFragment());
assertEquals("frag1", builder.getFragment());
}
@Test
public void ofSpecialTest() {
//测试不规范的或者无需解码的字符串是否成功解码
final UrlBuilder builder = UrlBuilder.ofHttp(" www.hutool.cn/aaa/bbb/?a=张三&b=%%e5%9b%9b#frag1", CharsetUtil.CHARSET_UTF_8);
Assert.assertEquals("http", builder.getScheme());
Assert.assertEquals("www.hutool.cn", builder.getHost());
assertEquals("http", builder.getScheme());
assertEquals("www.hutool.cn", builder.getHost());
Assert.assertEquals("aaa", builder.getPath().getSegment(0));
Assert.assertEquals("bbb", builder.getPath().getSegment(1));
assertEquals("aaa", builder.getPath().getSegment(0));
assertEquals("bbb", builder.getPath().getSegment(1));
Assert.assertEquals("张三", builder.getQuery().get("a"));
Assert.assertEquals("%四", builder.getQuery().get("b"));
assertEquals("张三", builder.getQuery().get("a"));
assertEquals("%四", builder.getQuery().get("b"));
Assert.assertEquals("frag1", builder.getFragment());
assertEquals("frag1", builder.getFragment());
}
@Test
@ -217,7 +221,7 @@ public class UrlBuilderTest {
"&chksm=6cbda3a25bca2ab4516410db6ce6e125badaac2f8c5548ea6e18eab6dc3c5422cb8cbe1095f7";
final UrlBuilder builder = UrlBuilder.ofHttp(urlStr, CharsetUtil.CHARSET_UTF_8);
// 原URL中的&替换为&
Assert.assertEquals("https://mp.weixin.qq.com/s?" +
assertEquals("https://mp.weixin.qq.com/s?" +
"__biz=MzI5NjkyNTIxMg==" +
"&mid=100000465&idx=1" +
"&sn=1044c0d19723f74f04f4c1da34eefa35" +
@ -231,40 +235,40 @@ public class UrlBuilderTest {
final String today = DateUtil.date().toString("yyyyMMdd");
final String getWorkDayUrl = "https://tool.bitefu.net/jiari/?info=1&d=" + today;
final UrlBuilder builder = UrlBuilder.ofHttp(getWorkDayUrl, CharsetUtil.CHARSET_UTF_8);
Assert.assertEquals(getWorkDayUrl, builder.toString());
assertEquals(getWorkDayUrl, builder.toString());
}
@Test
public void blankEncodeTest() {
final UrlBuilder urlBuilder = UrlBuilder.ofHttp("http://a.com/aaa bbb.html", CharsetUtil.CHARSET_UTF_8);
Assert.assertEquals("http://a.com/aaa%20bbb.html", urlBuilder.toString());
assertEquals("http://a.com/aaa%20bbb.html", urlBuilder.toString());
}
@Test
public void dotEncodeTest() {
final UrlBuilder urlBuilder = UrlBuilder.ofHttp("http://xtbgyy.digitalgd.com.cn/ebus/../../..", CharsetUtil.CHARSET_UTF_8);
Assert.assertEquals("http://xtbgyy.digitalgd.com.cn/ebus/../../..", urlBuilder.toString());
assertEquals("http://xtbgyy.digitalgd.com.cn/ebus/../../..", urlBuilder.toString());
}
@Test
public void multiSlashTest() {
//issue#I25MZL某些URL中有多个斜杠此为合法路径
final UrlBuilder urlBuilder = UrlBuilder.ofHttp("https://hutool.cn//file/test.jpg", CharsetUtil.CHARSET_UTF_8);
Assert.assertEquals("https://hutool.cn//file/test.jpg", urlBuilder.toString());
assertEquals("https://hutool.cn//file/test.jpg", urlBuilder.toString());
}
@Test
public void toURITest() throws URISyntaxException {
final String webUrl = "http://exmple.com/patha/pathb?a=123"; // 报错数据
final UrlBuilder urlBuilder = UrlBuilder.of(webUrl, StandardCharsets.UTF_8);
Assert.assertEquals(new URI(webUrl), urlBuilder.toURI());
assertEquals(new URI(webUrl), urlBuilder.toURI());
}
@Test
public void testEncodeInQuery() {
final String webUrl = "http://exmple.com/patha/pathb?a=123&b=4?6&c=789"; // b=4?6 参数中有未编码的
final UrlBuilder urlBuilder = UrlBuilder.of(webUrl, StandardCharsets.UTF_8);
Assert.assertEquals("a=123&b=4?6&c=789", urlBuilder.getQueryStr());
assertEquals("a=123&b=4?6&c=789", urlBuilder.getQueryStr());
}
@Test
@ -272,7 +276,7 @@ public class UrlBuilderTest {
// Path中的某些符号无需转义比如=
final String urlStr = "http://hq.sinajs.cn/list=sh600519";
final UrlBuilder urlBuilder = UrlBuilder.ofHttp(urlStr, CharsetUtil.CHARSET_UTF_8);
Assert.assertEquals(urlStr, urlBuilder.toString());
assertEquals(urlStr, urlBuilder.toString());
}
@Test
@ -281,7 +285,7 @@ public class UrlBuilderTest {
// Path中`:`在第一个segment需要转义之后的不需要
final String urlStr = "https://hutool.cn/aa/bb/Pre-K,Kindergarten,First,Second,Third,Fourth,Fifth/Page:3";
final UrlBuilder urlBuilder = UrlBuilder.ofHttp(urlStr, CharsetUtil.CHARSET_UTF_8);
Assert.assertEquals(urlStr, urlBuilder.toString());
assertEquals(urlStr, urlBuilder.toString());
}
@Test
@ -291,7 +295,7 @@ public class UrlBuilderTest {
// PATH除了第一个path外:是允许的
final String url2 = "https://gimg2.baidu.com/image_search/src=http:%2F%2Fpic.jj20.com%2Fup%2Fallimg%2F1114%2F0H320120Z3%2F200H3120Z3-6-1200.jpg&refer=http:%2F%2Fpic.jj20.com&app=2002&size=f9999,10000&q=a80&n=0&g=0n&fmt=jpeg?sec=1621996490&t=8c384c2823ea453da15a1b9cd5183eea";
Assert.assertEquals(url2, urlBuilder.toString());
assertEquals(url2, urlBuilder.toString());
}
@Test
@ -300,10 +304,10 @@ public class UrlBuilderTest {
// https://stackoverflow.com/questions/26088849/url-fragment-allowed-characters
final String url = "https://hutool.cn/docs/#/?id=简介";
UrlBuilder urlBuilder = UrlBuilder.ofHttp(url);
Assert.assertEquals("https://hutool.cn/docs/#/?id=%E7%AE%80%E4%BB%8B", urlBuilder.toString());
assertEquals("https://hutool.cn/docs/#/?id=%E7%AE%80%E4%BB%8B", urlBuilder.toString());
urlBuilder = UrlBuilder.ofHttp(urlBuilder.toString());
Assert.assertEquals(urlBuilder.toString(), urlBuilder.toString());
assertEquals(urlBuilder.toString(), urlBuilder.toString());
}
@Test
@ -313,7 +317,7 @@ public class UrlBuilderTest {
// https://www.rfc-editor.org/rfc/rfc3986.html#section-3.4
final String url = "https://invoice.maycur.com/2b27a802-8423-4d41-86f5-63a6b259f61e.xlsx?download/2b27a802-8423-4d41-86f5-63a6b259f61e.xlsx&e=1630491088";
final UrlBuilder urlBuilder = UrlBuilder.ofHttp(url);
Assert.assertEquals(url, urlBuilder.toString());
assertEquals(url, urlBuilder.toString());
}
@Test
@ -326,7 +330,7 @@ public class UrlBuilderTest {
.addPath("bbb")
.build();
Assert.assertEquals("https://domain.cn/api/xxx/bbb", url);
assertEquals("https://domain.cn/api/xxx/bbb", url);
}
@Test
@ -338,21 +342,21 @@ public class UrlBuilderTest {
.addPath("/api/xxx/bbb")
.build();
Assert.assertEquals("https://domain.cn/api/xxx/bbb", url);
assertEquals("https://domain.cn/api/xxx/bbb", url);
}
@Test
public void percent2BTest() {
final String url = "http://xxx.cn/a?Signature=3R013Bj9Uq4YeISzAs2iC%2BTVCL8%3D";
final UrlBuilder of = UrlBuilder.ofHttpWithoutEncode(url);
Assert.assertEquals(url, of.toString());
assertEquals(url, of.toString());
}
@Test
public void paramTest() {
final String url = "http://ci.xiaohongshu.com/spectrum/c136c98aa2047babe25b994a26ffa7b492bd8058?imageMogr2/thumbnail/x800/format/jpg";
final UrlBuilder builder = UrlBuilder.ofHttp(url);
Assert.assertEquals(url, builder.toString());
assertEquals(url, builder.toString());
}
@Test
@ -361,7 +365,7 @@ public class UrlBuilderTest {
final String url = "https://www.hutool.cn/#/a/b?timestamp=1640391380204";
final UrlBuilder builder = UrlBuilder.ofHttp(url);
Assert.assertEquals(url, builder.toString());
assertEquals(url, builder.toString());
}
@Test
@ -370,7 +374,7 @@ public class UrlBuilderTest {
final String url = "https://www.hutool.cn/#/a/b";
final UrlBuilder builder = UrlBuilder.ofHttp(url);
builder.setFragment(builder.getFragment() + "?timestamp=1640391380204");
Assert.assertEquals("https://www.hutool.cn/#/a/b?timestamp=1640391380204", builder.toString());
assertEquals("https://www.hutool.cn/#/a/b?timestamp=1640391380204", builder.toString());
}
@Test
@ -385,7 +389,7 @@ public class UrlBuilderTest {
"/RqAAYRYVCBiyuzAexSiDiJX1VqWljg4jYp1sdyv3HpV3sXVcf6VH6AN9ot5YNTw4JNO0aNpLpLm93rRMrOKIOsve+OmNyZ4HS7qHQKt1qp7HY1A" +
"/wGhJstkAoGQt+CHSMwVdIx3bVT1+ZYnJdM/oIQ/90afw4EEEQaRE51Z0rQC7z8d";
final String build = UrlBuilder.of(url).build();
Assert.assertEquals(url, build);
assertEquals(url, build);
}
@Test
@ -397,21 +401,21 @@ public class UrlBuilderTest {
"&Signature=oYUu26JufAyPY4PdzaOp1x4sr4Q%3D";
final UrlBuilder urlBuilder = UrlBuilder.ofHttp(url, null);
Assert.assertEquals(url, urlBuilder.toString());
assertEquals(url, urlBuilder.toString());
}
@Test
public void issue2215Test() {
final String url = "https://hutool.cn/v1/104303371/messages:send";
final String build = UrlBuilder.of(url).build();
Assert.assertEquals(url, build);
assertEquals(url, build);
}
@Test
public void issuesI4Z2ETTest() {
final String url = "http://hutool.cn/2022/03/09/123.zip?Expires=1648704684&OSSAccessKeyId=LTAI4FncgaVtwZGBnYHHi8ox&Signature=%2BK%2B%3D";
final String build = UrlBuilder.of(url, null).build();
Assert.assertEquals(url, build);
assertEquals(url, build);
}
@Test
@ -425,7 +429,7 @@ public class UrlBuilderTest {
final UrlBuilder builder = UrlBuilder.of(url);
params.forEach(builder::addQuery);
Assert.assertEquals("http://127.0.0.1/devicerecord/list?start=2022-03-31%2000:00:00&end=2022-03-31%2023:59:59&page=1&limit=10", builder.toString());
assertEquals("http://127.0.0.1/devicerecord/list?start=2022-03-31%2000:00:00&end=2022-03-31%2023:59:59&page=1&limit=10", builder.toString());
}
@Test
@ -439,7 +443,7 @@ public class UrlBuilderTest {
// 如果用户已经做了%编码不应该重复编码
final String url = "https://hutool.cn/v1.0?privateNum=%2B8616512884988";
final String s = UrlBuilder.of(url, null).setCharset(CharsetUtil.CHARSET_UTF_8).toString();
Assert.assertEquals(url, s);
assertEquals(url, s);
}
@Test
@ -447,7 +451,7 @@ public class UrlBuilderTest {
// &自动转换为&
final String url = "https://hutool.cn/a.mp3?Expires=1652423884&key=JMv2rKNc7Pz&sign=12zva00BpVqgZcX1wcb%2BrmN7H3E%3D";
final UrlBuilder of = UrlBuilder.of(url, null);
Assert.assertEquals(url.replace("&", "&"), of.toString());
assertEquals(url.replace("&", "&"), of.toString());
}
@SuppressWarnings("ConstantConditions")
@ -457,14 +461,14 @@ public class UrlBuilderTest {
.addQuery("param[0].field", "编码")
.toURI()
.toString();
Assert.assertEquals("http://127.0.0.1:8080?param%5B0%5D.field=%E7%BC%96%E7%A0%81", duplicate);
assertEquals("http://127.0.0.1:8080?param%5B0%5D.field=%E7%BC%96%E7%A0%81", duplicate);
String normal = UrlBuilder.ofHttp("127.0.0.1:8080")
.addQuery("param[0].field", "编码")
.toURL()
.toURI()
.toString();
Assert.assertEquals(duplicate, normal);
assertEquals(duplicate, normal);
}
@Test
@ -475,4 +479,13 @@ public class UrlBuilderTest {
UrlBuilder.of().addPath("//");
UrlBuilder.of().addPath("//a");
}
@Test
public void pathWithColonTest() {
// https://www.ietf.org/rfc/rfc3986.html#section-3.3
// 此处Path中是允许有`:`之前理解有误应该是相对URI的第一个segment中不允许有`:`
final UrlPath httpUrl = UrlPath.of("/a:1/b:1/c:1/d:1/", CharsetUtil.CHARSET_UTF_8);
final String build = httpUrl.build(CharsetUtil.CHARSET_UTF_8);
assertEquals("/a:1/b:1/c:1/d:1/", build);
}
}

View File

@ -242,6 +242,13 @@ public class HttpRequestTest {
Console.log(httpResponse.body());
}
@Test
@Ignore
public void issueIAAE88Test() {
final HttpRequest request = HttpRequest.of("http://127.0.0.1:8003/com.rnd.aiq:message/message/getName/15", null);
Console.log(request);
}
@Test
@Ignore
public void testHttpResource() {