texture_filters: update ScaleForce (#5270)

* texture_filters: update ScaleForce

* texture_filters: optimize scale_force

* texture_filters/scale_force: optimize final offset calculation

src/video_core/renderer_opengl/texture_filters/scale_force/scale_force.frag

@@ -1,7 +1,6 @@
 //? #version 320 es
 
 // from https://github.com/BreadFish64/ScaleFish/tree/master/scale_force
-// shader adapted to GLSL 320 es and debugging outputs stripped
 
 // MIT License
 //
@@ -27,15 +26,18 @@
 
 precision mediump float;
 
-in highp vec2 tex_coord;
+in vec2 tex_coord;
 
-out mediump vec4 frag_color;
+out vec4 frag_color;
 
 uniform sampler2D input_texture;
 
+vec2 tex_size;
+vec2 inv_tex_size;
+
 vec4 cubic(float v) {
-    vec4 n = vec4(1.0, 2.0, 3.0, 4.0) - v;
-    vec4 s = n * n * n;
+    vec3 n = vec3(1.0, 2.0, 3.0) - v;
+    vec3 s = n * n * n;
     float x = s.x;
     float y = s.y - 4.0 * s.x;
     float z = s.z - 4.0 * s.y + 6.0 * s.x;
@@ -43,14 +45,11 @@ vec4 cubic(float v) {
     return vec4(x, y, z, w) / 6.0;
 }
 
-vec4 textureBicubic(sampler2D sampler, vec2 tex_coords) {
-    vec2 tex_size = vec2(textureSize(sampler, 0));
-    vec2 inv_tex_size = 1.0 / tex_size;
-
+// Bicubic interpolation
+vec4 textureBicubic(vec2 tex_coords) {
     tex_coords = tex_coords * tex_size - 0.5;
 
-    vec2 fxy = fract(tex_coords);
-    tex_coords -= fxy;
+    vec2 fxy = modf(tex_coords, tex_coords);
 
     vec4 xcubic = cubic(fxy.x);
     vec4 ycubic = cubic(fxy.y);
@@ -62,10 +61,10 @@ vec4 textureBicubic(sampler2D sampler, vec2 tex_coords) {
 
     offset *= inv_tex_size.xxyy;
 
-    vec4 sample0 = texture(sampler, offset.xz);
-    vec4 sample1 = texture(sampler, offset.yz);
-    vec4 sample2 = texture(sampler, offset.xw);
-    vec4 sample3 = texture(sampler, offset.yw);
+    vec4 sample0 = textureLod(input_texture, offset.xz, 0.0);
+    vec4 sample1 = textureLod(input_texture, offset.yz, 0.0);
+    vec4 sample2 = textureLod(input_texture, offset.xw, 0.0);
+    vec4 sample3 = textureLod(input_texture, offset.yw, 0.0);
 
     float sx = s.x / (s.x + s.y);
     float sy = s.z / (s.z + s.w);
@@ -73,40 +72,66 @@ vec4 textureBicubic(sampler2D sampler, vec2 tex_coords) {
     return mix(mix(sample3, sample2, sx), mix(sample1, sample0, sx), sy);
 }
 
-float ColorDist(vec4 a, vec4 b) {
+mat4x3 center_matrix;
+vec4 center_alpha;
+
+// Finds the distance between four colors and cc in YCbCr space
+vec4 ColorDist(vec4 A, vec4 B, vec4 C, vec4 D) {
     // https://en.wikipedia.org/wiki/YCbCr#ITU-R_BT.2020_conversion
     const vec3 K = vec3(0.2627, 0.6780, 0.0593);
-    const float luminance_weight = .6;
-    const mat3 MATRIX = mat3(K * luminance_weight, -.5 * K.r / (1.0 - K.b), -.5 * K.g / (1.0 - K.b),
-                             .5, .5, -.5 * K.g / (1.0 - K.r), -.5 * K.b / (1.0 - K.r));
-    vec4 diff = a - b;
-    vec3 YCbCr = diff.rgb * MATRIX;
-    float d = length(YCbCr) * length(vec3(1.0)) / length(vec3(luminance_weight, 1.0, 1.0));
-    return sqrt(a.a * b.a * d * d + diff.a * diff.a);
-}
+    const float LUMINANCE_WEIGHT = .6;
+    const mat3 YCBCR_MATRIX =
+        mat3(K * LUMINANCE_WEIGHT, -.5 * K.r / (1.0 - K.b), -.5 * K.g / (1.0 - K.b), .5, .5,
+             -.5 * K.g / (1.0 - K.r), -.5 * K.b / (1.0 - K.r));
 
-const int radius = 2;
+    mat4x3 colors = mat4x3(A.rgb, B.rgb, C.rgb, D.rgb) - center_matrix;
+    mat4x3 YCbCr = YCBCR_MATRIX * colors;
+    vec4 color_dist = vec3(1.0) * YCbCr;
+    color_dist *= color_dist;
+    vec4 alpha = vec4(A.a, B.a, C.a, D.a);
+
+    return sqrt((color_dist + abs(center_alpha - alpha)) * alpha * center_alpha);
+}
 
 void main() {
-    vec2 input_size = vec2(textureSize(input_texture, 0));
-    vec4 center_texel = texture(input_texture, tex_coord);
-    vec2 final_offset = vec2(0.0);
-    float total_diff = 0.0;
+    vec4 bl = textureLodOffset(input_texture, tex_coord, 0.0, ivec2(-1, -1));
+    vec4 bc = textureLodOffset(input_texture, tex_coord, 0.0, ivec2(0, -1));
+    vec4 br = textureLodOffset(input_texture, tex_coord, 0.0, ivec2(1, -1));
+    vec4 cl = textureLodOffset(input_texture, tex_coord, 0.0, ivec2(-1, 0));
+    vec4 cc = textureLod(input_texture, tex_coord, 0.0);
+    vec4 cr = textureLodOffset(input_texture, tex_coord, 0.0, ivec2(1, 0));
+    vec4 tl = textureLodOffset(input_texture, tex_coord, 0.0, ivec2(-1, 1));
+    vec4 tc = textureLodOffset(input_texture, tex_coord, 0.0, ivec2(0, 1));
+    vec4 tr = textureLodOffset(input_texture, tex_coord, 0.0, ivec2(1, 1));
 
-    for (int y = -radius; y <= radius; ++y) {
-        for (int x = -radius; x <= radius; ++x) {
-            if (0 == (x | y))
-                continue;
-            vec2 offset = vec2(x, y);
-            float weight = pow(length(offset), -length(offset));
-            vec4 texel = texture(input_texture, tex_coord + offset / input_size);
-            float diff = ColorDist(texel, center_texel) * weight;
-            total_diff += diff;
-            final_offset += diff * offset;
-        }
+
+    tex_size = vec2(textureSize(input_texture, 0));
+    inv_tex_size = 1.0 / tex_size;
+    center_matrix = mat4x3(cc.rgb, cc.rgb, cc.rgb, cc.rgb);
+    center_alpha = cc.aaaa;
+
+    vec4 offset_tl = ColorDist(tl, tc, tr, cr);
+    vec4 offset_br = ColorDist(br, bc, bl, cl);
+
+    // Calculate how different cc is from the texels around it
+    float total_dist = dot(offset_tl + offset_br, vec4(1.0));
+
+    // Add together all the distances with direction taken into account
+    vec4 tmp = offset_tl - offset_br;
+    vec2 total_offset = tmp.wy + tmp.zz + vec2(-tmp.x, tmp.x);
+
+    if (total_dist == 0.0) {
+        // Doing bicubic filtering just past the edges where the offset is 0 causes black floaters
+        // and it doesn't really matter which filter is used when the colors aren't changing.
+        frag_color = cc;
+    } else {
+        // When the image has thin points, they tend to split apart.
+        // This is because the texels all around are different
+        // and total_offset reaches into clear areas.
+        // This works pretty well to keep the offset in bounds for these cases.
+        float clamp_val = length(total_offset) / total_dist;
+        vec2 final_offset = clamp(total_offset, -clamp_val, clamp_val) * inv_tex_size;
+
+        frag_color = textureBicubic(tex_coord - final_offset);
     }
-
-    float clamp_val = length(final_offset) / total_diff;
-    final_offset = clamp(final_offset, -clamp_val, clamp_val);
-    frag_color = textureBicubic(input_texture, tex_coord - final_offset / input_size);
 }