blob: df290ded753b01393881d70ba1ce780380737102 [file] [log] [blame]
/*
* Copyright 2017 Google Inc.
*
* Use of this source code is governed by a BSD-style license that can be
* found in the LICENSE file.
*/
#include "SkColorPriv.h"
#include "SkCpu.h"
#include "SkJumper.h"
#include "SkOnce.h"
#include "SkRasterPipeline.h"
#include "SkTemplates.h"
// We'll use __has_feature(memory_sanitizer) to detect MSAN.
// SkJumper_generated.S is not compiled with MSAN, so MSAN would yell really loud.
#if !defined(__has_feature)
#define __has_feature(x) 0
#endif
// Stages expect these constants to be set to these values.
// It's fine to rearrange and add new ones if you update SkJumper_constants.
using K = const SkJumper_constants;
static K kConstants = {
{0,1,2,3,4,5,6,7},
{0,1,2,3,4,5,6,7},
};
#define M(st) +1
static const int kNumStages = SK_RASTER_PIPELINE_STAGES(M);
#undef M
#ifndef SK_DISABLE_SSSE3_RUNTIME_CHECK_FOR_LOWP_STAGES
#if 0 && !__has_feature(memory_sanitizer) && (defined(__x86_64__) || defined(_M_X64))
#include <atomic>
#define M(st) #st,
static const char* kStageNames[] = { SK_RASTER_PIPELINE_STAGES(M) };
#undef M
static std::atomic<int> gMissingStageCounters[kNumStages];
static void log_missing(SkRasterPipeline::StockStage st) {
static SkOnce once;
once([] { atexit([] {
for (int i = 0; i < kNumStages; i++) {
if (int count = gMissingStageCounters[i].load()) {
SkDebugf("%7d\t%s\n", count, kStageNames[i]);
}
}
}); });
gMissingStageCounters[st]++;
}
#else
static void log_missing(SkRasterPipeline::StockStage) {}
#endif
#endif
// We can't express the real types of most stage functions portably, so we use a stand-in.
// We'll only ever call start_pipeline() or start_pipeline_2d(), which then chain into the rest.
using StageFn = void(void);
using StartPipelineFn = void(size_t,size_t,size_t, void**,K*);
using StartPipeline2dFn = void(size_t,size_t,size_t,size_t, void**,K*);
// Some platforms expect C "name" maps to asm "_name", others to "name".
#if defined(__APPLE__)
#define ASM(name, suffix) sk_##name##_##suffix
#else
#define ASM(name, suffix) _sk_##name##_##suffix
#endif
// Some stages have low-precision (~15 bit) versions from SkJumper_stages_lowp.cpp.
#define LOWP_STAGES(M) \
M(black_color) M(white_color) M(uniform_color) \
M(set_rgb) \
M(premul) \
M(load_8888) M(load_8888_dst) M(store_8888) \
M(load_bgra) M(load_bgra_dst) M(store_bgra) \
M(load_a8) M(load_a8_dst) M(store_a8) \
M(load_g8) M(load_g8_dst) \
M(swap_rb) \
M(srcover_rgba_8888) \
M(lerp_1_float) \
M(lerp_u8) \
M(scale_1_float) \
M(scale_u8) \
M(move_src_dst) \
M(move_dst_src) \
M(clear) \
M(srcatop) \
M(dstatop) \
M(srcin) \
M(dstin) \
M(srcout) \
M(dstout) \
M(srcover) \
M(dstover) \
M(modulate) \
M(multiply) \
M(screen) \
M(xor_)
extern "C" {
#if __has_feature(memory_sanitizer) || defined(STARBOARD)
// We'll just run portable code.
#elif defined(__aarch64__)
StartPipelineFn ASM(start_pipeline ,aarch64);
StartPipeline2dFn ASM(start_pipeline_2d,aarch64);
StageFn ASM(just_return,aarch64);
#define M(st) StageFn ASM(st,aarch64);
SK_RASTER_PIPELINE_STAGES(M)
#undef M
#elif defined(__arm__)
StartPipelineFn ASM(start_pipeline ,vfp4);
StartPipeline2dFn ASM(start_pipeline_2d,vfp4);
StageFn ASM(just_return,vfp4);
#define M(st) StageFn ASM(st,vfp4);
SK_RASTER_PIPELINE_STAGES(M)
#undef M
#elif defined(__x86_64__) || defined(_M_X64)
StartPipelineFn ASM(start_pipeline,hsw ),
ASM(start_pipeline,avx ),
ASM(start_pipeline,sse41 ),
ASM(start_pipeline,sse2 ),
ASM(start_pipeline,hsw_lowp ),
ASM(start_pipeline,ssse3_lowp);
StartPipeline2dFn ASM(start_pipeline_2d,hsw ),
ASM(start_pipeline_2d,avx ),
ASM(start_pipeline_2d,sse41 ),
ASM(start_pipeline_2d,sse2 ),
ASM(start_pipeline_2d,hsw_lowp ),
ASM(start_pipeline_2d,ssse3_lowp);
StageFn ASM(just_return,hsw),
ASM(just_return,avx),
ASM(just_return,sse41),
ASM(just_return,sse2),
ASM(just_return,hsw_lowp ),
ASM(just_return,ssse3_lowp);
#define M(st) StageFn ASM(st,hsw);
SK_RASTER_PIPELINE_STAGES(M)
#undef M
#define M(st) StageFn ASM(st,avx);
SK_RASTER_PIPELINE_STAGES(M)
#undef M
#define M(st) StageFn ASM(st,sse41);
SK_RASTER_PIPELINE_STAGES(M)
#undef M
#define M(st) StageFn ASM(st,sse2);
SK_RASTER_PIPELINE_STAGES(M)
#undef M
#define M(st) StageFn ASM(st,hsw_lowp);
LOWP_STAGES(M)
#undef M
#define M(st) StageFn ASM(st,ssse3_lowp);
LOWP_STAGES(M)
#undef M
#elif (defined(__i386__) || defined(_M_IX86)) && \
!(defined(_MSC_VER) && defined(SK_SUPPORT_LEGACY_WIN32_JUMPER))
StartPipelineFn ASM(start_pipeline ,sse2);
StartPipeline2dFn ASM(start_pipeline_2d,sse2);
StageFn ASM(just_return,sse2);
#define M(st) StageFn ASM(st,sse2);
SK_RASTER_PIPELINE_STAGES(M)
#undef M
#endif
// Portable, single-pixel stages.
StartPipelineFn sk_start_pipeline;
StartPipeline2dFn sk_start_pipeline_2d;
StageFn sk_just_return;
#define M(st) StageFn sk_##st;
SK_RASTER_PIPELINE_STAGES(M)
#undef M
}
#if !defined(STARBOARD)
#if !__has_feature(memory_sanitizer) && (defined(__x86_64__) || defined(_M_X64))
template <SkRasterPipeline::StockStage st>
static constexpr StageFn* hsw_lowp() { return nullptr; }
template <SkRasterPipeline::StockStage st>
static constexpr StageFn* ssse3_lowp() { return nullptr; }
#define M(st) \
template <> constexpr StageFn* hsw_lowp<SkRasterPipeline::st>() { \
return ASM(st,hsw_lowp); \
} \
template <> constexpr StageFn* ssse3_lowp<SkRasterPipeline::st>() { \
return ASM(st,ssse3_lowp); \
}
LOWP_STAGES(M)
#undef M
#endif
#endif // !defined(STARBOARD)
// Engines comprise everything we need to run SkRasterPipelines.
struct SkJumper_Engine {
StageFn* stages[kNumStages];
StartPipelineFn* start_pipeline;
StartPipeline2dFn* start_pipeline_2d;
StageFn* just_return;
};
// We'll default to this portable engine, but try to choose a better one at runtime.
static const SkJumper_Engine kPortable = {
#define M(stage) sk_##stage,
{ SK_RASTER_PIPELINE_STAGES(M) },
#undef M
sk_start_pipeline,
sk_start_pipeline_2d,
sk_just_return,
};
static SkJumper_Engine gEngine = kPortable;
static SkOnce gChooseEngineOnce;
static SkJumper_Engine choose_engine() {
#if __has_feature(memory_sanitizer) || defined(STARBOARD)
// We'll just run portable code.
#elif defined(__aarch64__)
return {
#define M(stage) ASM(stage, aarch64),
{ SK_RASTER_PIPELINE_STAGES(M) },
M(start_pipeline)
M(start_pipeline_2d)
M(just_return)
#undef M
};
#elif defined(__arm__)
if (1 && SkCpu::Supports(SkCpu::NEON|SkCpu::NEON_FMA|SkCpu::VFP_FP16)) {
return {
#define M(stage) ASM(stage, vfp4),
{ SK_RASTER_PIPELINE_STAGES(M) },
M(start_pipeline)
M(start_pipeline_2d)
M(just_return)
#undef M
};
}
#elif defined(__x86_64__) || defined(_M_X64)
if (1 && SkCpu::Supports(SkCpu::HSW)) {
return {
#define M(stage) ASM(stage, hsw),
{ SK_RASTER_PIPELINE_STAGES(M) },
M(start_pipeline)
M(start_pipeline_2d)
M(just_return)
#undef M
};
}
if (1 && SkCpu::Supports(SkCpu::AVX)) {
return {
#define M(stage) ASM(stage, avx),
{ SK_RASTER_PIPELINE_STAGES(M) },
M(start_pipeline)
M(start_pipeline_2d)
M(just_return)
#undef M
};
}
if (1 && SkCpu::Supports(SkCpu::SSE41)) {
return {
#define M(stage) ASM(stage, sse41),
{ SK_RASTER_PIPELINE_STAGES(M) },
M(start_pipeline)
M(start_pipeline_2d)
M(just_return)
#undef M
};
}
if (1 && SkCpu::Supports(SkCpu::SSE2)) {
return {
#define M(stage) ASM(stage, sse2),
{ SK_RASTER_PIPELINE_STAGES(M) },
M(start_pipeline)
M(start_pipeline_2d)
M(just_return)
#undef M
};
}
#elif (defined(__i386__) || defined(_M_IX86)) && \
!(defined(_MSC_VER) && defined(SK_SUPPORT_LEGACY_WIN32_JUMPER))
if (1 && SkCpu::Supports(SkCpu::SSE2)) {
return {
#define M(stage) ASM(stage, sse2),
{ SK_RASTER_PIPELINE_STAGES(M) },
M(start_pipeline)
M(start_pipeline_2d)
M(just_return)
#undef M
};
}
#endif
return kPortable;
}
#ifndef SK_DISABLE_SSSE3_RUNTIME_CHECK_FOR_LOWP_STAGES
static const SkJumper_Engine kNone = {
#define M(stage) nullptr,
{ SK_RASTER_PIPELINE_STAGES(M) },
#undef M
nullptr,
nullptr,
nullptr,
};
static SkJumper_Engine gLowp = kNone;
static SkOnce gChooseLowpOnce;
static SkJumper_Engine choose_lowp() {
#if !defined(STARBOARD)
#if !__has_feature(memory_sanitizer) && (defined(__x86_64__) || defined(_M_X64))
if (1 && SkCpu::Supports(SkCpu::HSW)) {
return {
#define M(st) hsw_lowp<SkRasterPipeline::st>(),
{ SK_RASTER_PIPELINE_STAGES(M) },
ASM(start_pipeline ,hsw_lowp),
ASM(start_pipeline_2d,hsw_lowp),
ASM(just_return ,hsw_lowp)
#undef M
};
}
if (1 && SkCpu::Supports(SkCpu::SSSE3)) {
return {
#define M(st) ssse3_lowp<SkRasterPipeline::st>(),
{ SK_RASTER_PIPELINE_STAGES(M) },
ASM(start_pipeline ,ssse3_lowp),
ASM(start_pipeline_2d,ssse3_lowp),
ASM(just_return ,ssse3_lowp)
#undef M
};
}
#endif
#endif // !defined(STARBOARD)
return kNone;
}
#endif
const SkJumper_Engine& SkRasterPipeline::build_pipeline(void** ip) const {
#ifndef SK_DISABLE_SSSE3_RUNTIME_CHECK_FOR_LOWP_STAGES
gChooseLowpOnce([]{ gLowp = choose_lowp(); });
// First try to build a lowp pipeline. If that fails, fall back to normal float gEngine.
void** reset_point = ip;
*--ip = (void*)gLowp.just_return;
for (const StageList* st = fStages; st; st = st->prev) {
if (st->stage == SkRasterPipeline::clamp_0) {
continue; // No-op in lowp.
}
if (StageFn* fn = gLowp.stages[st->stage]) {
if (st->ctx) {
*--ip = st->ctx;
}
*--ip = (void*)fn;
} else {
log_missing(st->stage);
ip = reset_point;
break;
}
}
if (ip != reset_point) {
return gLowp;
}
#endif
gChooseEngineOnce([]{ gEngine = choose_engine(); });
// We're building the pipeline backwards, so we start with the final stage just_return.
*--ip = (void*)gEngine.just_return;
// Still going backwards, each stage's context pointer then its StageFn.
for (const StageList* st = fStages; st; st = st->prev) {
if (st->ctx) {
*--ip = st->ctx;
}
*--ip = (void*)gEngine.stages[st->stage];
}
return gEngine;
}
void SkRasterPipeline::run(size_t x, size_t y, size_t n) const {
if (this->empty()) {
return;
}
// Best to not use fAlloc here... we can't bound how often run() will be called.
SkAutoSTMalloc<64, void*> program(fSlotsNeeded);
const SkJumper_Engine& engine = this->build_pipeline(program.get() + fSlotsNeeded);
engine.start_pipeline(x,y,x+n, program.get(), &kConstants);
}
std::function<void(size_t, size_t, size_t)> SkRasterPipeline::compile() const {
if (this->empty()) {
return [](size_t, size_t, size_t) {};
}
void** program = fAlloc->makeArray<void*>(fSlotsNeeded);
const SkJumper_Engine& engine = this->build_pipeline(program + fSlotsNeeded);
auto start_pipeline = engine.start_pipeline;
return [=](size_t x, size_t y, size_t n) {
start_pipeline(x,y,x+n, program, &kConstants);
};
}
void SkRasterPipeline::run_2d(size_t x, size_t y, size_t w, size_t h) const {
if (this->empty()) {
return;
}
// Like in run(), it's best to not use fAlloc here... we can't bound how often we'll be called.
SkAutoSTMalloc<64, void*> program(fSlotsNeeded);
const SkJumper_Engine& engine = this->build_pipeline(program.get() + fSlotsNeeded);
engine.start_pipeline_2d(x,y,x+w,y+h, program.get(), &kConstants);
}