Merged from master
jump to
@@ -1,43 +1,17 @@
-# Copied from https://github.com/nim-lang/Nim/wiki/TravisCI language: c -env: - # Build and test against the master and devel branches of Nim - - BRANCH=master - - BRANCH=devel -compiler: - #- gcc - - clang -matrix: - allow_failures: - # Ignore failures when building against the devel Nim branch - - env: BRANCH=devel - fast_finish: true +cache: + directories: + - "$HOME/.nimble" + - "$HOME/.choosenim" install: + - export CHOOSENIM_CHOOSE_VERSION="1.0.0" - | - if [ ! -x nim-$BRANCH/bin/nim ]; then - git clone -b $BRANCH --depth 1 git://github.com/nim-lang/nim nim-$BRANCH/ - cd nim-$BRANCH - git clone -b master --depth 1 git://github.com/nim-lang/csources csources/ - cd csources - sh build.sh - cd .. - rm -rf csources - bin/nim c koch - ./koch boot -d:release - else - cd nim-$BRANCH - git fetch origin - if ! git merge FETCH_HEAD | grep "Already up-to-date"; then - bin/nim c koch - ./koch boot -d:release - fi - fi - cd .. + curl https://nim-lang.org/choosenim/init.sh -sSf > init.sh + sh init.sh -y before_script: - - export PATH="nim-$BRANCH/bin${PATH:+:$PATH}" + - set -e + - set -x + - export PATH=$HOME/.nimble/bin:$PATH + - export CHOOSENIM_NO_ANALYTICS=1 script: - - nim c --cc:$CC --verbosity:0 src/litestore.nim -cache: - directories: - - nim-master - - nim-devel + - nimble -y build
@@ -1,6 +1,6 @@
The MIT License (MIT) -Copyright (c) 2015-2018 Fabio Cevasco +Copyright (c) 2015-2019 Fabio Cevasco Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal@@ -19,4 +19,3 @@ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -
@@ -22,7 +22,7 @@ for page in ${pages[@]}
do (cat "${page}"; printf "\n\n") >> LiteStore_UserGuide.md done -hastyscribe --field/version:1.5.0 LiteStore_UserGuide.md +hastyscribe --field/version:1.5.1 LiteStore_UserGuide.md rm LiteStore_UserGuide.md mv LiteStore_UserGuide.htm .. cd ../..
@@ -1,6 +1,7 @@
import - ospaths, strutils + +from os import parentDir, `/` template thisModuleFile: string = instantiationInfo(fullPaths = true).filename@@ -20,20 +21,18 @@ license = pkgLicense
bin = @[pkgName] srcDir = "src" skipDirs = @["test"] +installExt = @["nim", "c", "h", "json", "ico"] # Dependencies -requires "nim >= 0.19.0", "https://github.com/h3rald/nim-jwt" +requires "nim >= 1.0.0", "https://github.com/h3rald/nim-jwt" # Build const parallel = "" #"--parallelBuild:1 --verbosity:3" compile = "nim c -d:release --threads:on" & " " & parallel - linux_x86 = "--cpu:i386 --os:linux" linux_x64 = "--cpu:amd64 --os:linux" - linux_arm = "--cpu:arm --os:linux" - windows_x86 = "--cpu:i386 --os:windows" windows_x64 = "--cpu:amd64 --os:windows" macosx_x64 = "" ls = "litestore"@@ -48,21 +47,12 @@
proc filename_for(os: string, arch: string): string = return "litestore" & "_v" & version & "_" & os & "_" & arch & ".zip" -task windows_x86_build, "Build LiteStore for Windows (x86)": - shell compile, windows_x86, ls_file - task windows_x64_build, "Build LiteStore for Windows (x64)": shell compile, windows_x64, ls_file task linux_x64_build, "Build LiteStore for Linux (x64)": shell compile, linux_x64, ls_file - -task linux_x86_build, "Build LiteStore for Linux (x86)": - shell compile, linux_x86, ls_file - -task linux_arm_build, "Build LiteStore for Linux (ARM)": - shell compile, linux_arm, ls_file - + task macosx_x64_build, "Build LiteStore for Mac OS X (x64)": shell compile, macosx_x64, ls_file@@ -74,10 +64,6 @@ cd "src"
if db.existsFile: db.rmFile shell "litestore -d:admin import" - echo "\n\n\n WINDOWS - x86:\n\n" - windows_x86_buildTask() - shell zip, "$1 $2 $3 $4" % [filename_for("windows", "x86"), ls & ".exe", doc, db] - shell "rm", ls & ".exe" echo "\n\n\n WINDOWS - x64:\n\n" windows_x64_buildTask() shell zip, "$1 $2 $3 $4" % [filename_for("windows", "x64"), ls & ".exe", doc, db]@@ -86,16 +72,9 @@ echo "\n\n\n LINUX - x64:\n\n"
linux_x64_buildTask() shell zip, "$1 $2 $3 $4" % [filename_for("linux", "x64"), ls, doc, db] shell "rm", ls - echo "\n\n\n LINUX - x86:\n\n" - linux_x86_buildTask() - shell zip, "$1 $2 $3 $4" % [filename_for("linux", "x86"), ls, doc, db] - shell "rm", ls - echo "\n\n\n LINUX - ARM:\n\n" - linux_arm_buildTask() - shell zip, "$1 $2 $3 $4" % [filename_for("linux", "arm"), ls, doc, db] - shell "rm", ls + echo "\n\n\n MAC OS X - x64:\n\n" macosx_x64_buildTask() shell zip, "$1 $2 $3 $4" % [filename_for("macosx", "x64"), ls, doc, db] - shell "rm", ls + shell "rm", ls echo "\n\n\n ALL DONE!"
@@ -1,11 +1,11 @@
(function(){ 'use strict'; var app = window.LS || (window.LS = {}); - + app.flash = m.prop(); app.system = {}; m.route.mode = "hash"; - + app.init = function(info){ app.system = info; app.system.v = app.system.version.match(/(v.+)$/)[1];@@ -24,5 +24,5 @@ "/search/:q/:page/:limit": app.search
}); }; Info.get().then(app.init); - + }());
@@ -2,12 +2,12 @@ (function(){
'use strict'; var app = window.LS || (window.LS = {}); var u = app.utils; - + app.doclist = {}; - + // Subcomponent app.doclist.panel = { - + /** * @typedef {Object} DoclistPanelConfig * @prop {string} id@@ -27,10 +27,10 @@ m("p", args.tags.map(function(tag){
return u.taglink({name: tag, key: u.guid()}); })) ]); - return m(".row.search-result", m(".col-md-12", [u.panel(obj)])); + return m(".row.search-result", m(".col-md-12", [u.panel(obj)])); } }; - + /** * @param {Function} ctrl * @param {Object} args@@ -44,7 +44,7 @@ var results = m(".row", [m(".col-md-12", args.items.map(function(item){
item.key = u.guid(); return m.component(app.doclist.panel, item); }))]); - + return m("section", [ m(".row", [args.title]), m(".row", [args.subtitle]),@@ -53,5 +53,5 @@ results,
m(".row.text-center", [u.paginator(args.querydata)]) ]); }; - + }());
@@ -9,7 +9,7 @@ app.document.vm.init = function() {
var vm = this; vm.id = m.prop(m.route.param("id")); vm.action = m.route.param("action"); - vm.readOnly = true; + vm.readOnly = true; vm.contentType = m.prop(""); vm.updatedTags = m.prop(""); vm.content = "";@@ -21,7 +21,7 @@ vm.ext = vm.id().match(/\.([a-z0-9]+)$/i)[1];
} catch(e) { vm.ext = ""; } - + // Retrieve single document & update relevant variables vm.getDoc = function(cb){ vm.doc = Doc.get(vm.id());@@ -37,7 +37,7 @@ }
} }, vm.flashError); }; - + // Reset some properties based on action switch (vm.action) { case 'create':@@ -52,7 +52,7 @@ case 'view':
vm.getDoc(); break; } - + // View document in editor vm.viewDocument = function(){ if (vm.ext === "md" && vm.id().match(new RegExp("^admin\/md\/"))) {@@ -62,14 +62,14 @@ } else {
m.route("/document/view/"+vm.id()); } }; - + // Set current document editable vm.edit = function(){ vm.editor.setReadOnly(false); vm.action = "edit"; vm.flash(""); }; - + // Save document vm.save = function(){ var doc = {};@@ -95,7 +95,7 @@ } else {
put(); } }; - + // Delete Document vm.delete = function(){ Doc.delete(vm.id()).then(function(){@@ -107,7 +107,7 @@ m.route("/info");
}); }, vm.flashError); }; - + // Cancel editing vm.cancel = function(){ if (vm.action === "create"){@@ -116,7 +116,7 @@ } else {
vm.viewDocument(); } }; - + // Patch document (update tags) vm.patch = function(){ var sysTags = vm.tags.filter(function(t){return /^\$/.test(t)});@@ -129,7 +129,7 @@ vm.viewDocument();
}); }, vm.flashError); }; - + // File uploader callbacks. var onSuccess = function(data){ vm.id(data.id);@@ -139,13 +139,13 @@ app.system = info;
vm.viewDocument(); }); }; - + var onFailure = function(data) { vm.flashError(data); }; var modalId = u.guid(); vm.uploader = u.uploader({docid: vm.id() || "", onSuccess: onSuccess, onFailure: onFailure, id: modalId}); - + // Populate tools based on current action vm.tools = function(){ if (app.system.read_only) {@@ -174,7 +174,7 @@ }
return tools; }; }; - + // Module main view app.document.main = function(){ var vm = app.document.vm;@@ -196,8 +196,8 @@ action: vm.patch,
actionText: "Update", content: m("div", [ m("input", { - type: "text", - class:"form-control", + type: "text", + class:"form-control", onchange: m.withAttr("value", vm.updatedTags), value: vm.updatedTags(), placeholder: "Enter comma-separated tags..."@@ -235,7 +235,7 @@ } else {
panelContent = m.component(app.editor, vm); } var title = m("span",[titleLeft, titleRight]); - + return m("div", [ vm.uploader, u.modal(deleteDialogCfg),@@ -244,6 +244,6 @@ m(".row", [u.toolbar({links: vm.tools()})]),
m(".row", [u.panel({title: title, content:panelContent})]) ]); }; - + u.layout(app.document); -}());+}());
@@ -1,5 +1,5 @@
(function(){ - 'use strict'; + 'use strict'; var app = window.LS || (window.LS = {}); app.editor = {};@@ -61,5 +61,5 @@ args.content = JSON.stringify(args.content);
} return m(".editor.panel.panal-default", {config: app.editor.config(args)}, args.content); }; - -}());+ +}());
@@ -16,11 +16,11 @@ vm.links = app.system.read_only ? m.prop([]) : m.prop([{action: vm.edit, title: "Edit", icon: "edit"}]);
}; app.guide.main = function(){ return m("article.row", [ - u.toolbar({links: app.guide.vm.links()}), + u.toolbar({links: app.guide.vm.links()}), m.trust(app.guide.vm.content()) ]); }; u.layout(app.guide); -}());+}());
@@ -18,7 +18,7 @@ vm.links = m.prop([{action: vm.view, title: "View Source", icon: "code"}]);
}; app.htmldoc.main = function(){ return m("article.row", [ - u.toolbar({links: app.htmldoc.vm.links()}), + u.toolbar({links: app.htmldoc.vm.links()}), m.trust(app.htmldoc.vm.content()) ]); };
@@ -15,8 +15,8 @@ } else {
return m("li", [m("span", title+": "), m("strong", content)]); } }; - var readonly = info.read_only ? m("span.label.label-success", "Yes") : m("span.label.label-danger", "No"); - var mirror = info.mount ? m("span.label.label-success", "Yes") : m("span.label.label-danger", "No"); + var readonly = info.read_only ? m("span.label.label-success", "Yes") : m("span.label.label-danger", "No"); + var mirror = info.mount ? m("span.label.label-success", "Yes") : m("span.label.label-danger", "No"); var infolist = m(".col-sm-6", [m("ul.list-unstyled", [ li("Version", info.version), li("Datastore Version", info.datastore_version),
@@ -20,7 +20,7 @@ Doc.search(vm.query, vm.offset, vm.limit).then(function(result){
vm.result(result); vm.total = result.total; vm.execTime = (result.execution_time*1000).toFixed(0); - }, vm.flashError); + }, vm.flashError); }; app.search.main = function(){ var vm = app.search.vm;
@@ -1,5 +1,5 @@
(function(){ - 'use strict'; + 'use strict'; var app = window.LS || (window.LS = {}); var u = app.utils;@@ -10,7 +10,7 @@ return function(element, isInitialized, context){
$(element).change(function(event){ obj.file(element.files[0]); if (obj.reader.readyState != 1) { - obj.reader.readAsDataURL(obj.file()); + obj.reader.readAsDataURL(obj.file()); } }); };@@ -21,7 +21,7 @@ var vm = this;
vm.docid = m.prop(args.docid); vm.file = m.prop(); - vm.id = args.id; + vm.id = args.id; vm.btnId = "#upload-"+vm.id+"-btn"; vm.modalId = "#upload-"+vm.id+"-modal"; vm.reader = new FileReader();@@ -46,14 +46,14 @@ doc.data = window.atob(doc.data);
} return Doc.put(doc, vm.file().type).then(args.onSuccess, args.onFailure); }; - + vm.cancel = function(){ $("input:file").val(''); }; - + return vm; }; - + app.uploader.view = function(ctrl, args){ var config = { title: "Upload Document",@@ -78,7 +78,7 @@ m("p.help-block", "Select a file to upload as document.")
]), m(".checkbox", [ m("label", [ - m("input", {type: "checkbox", value: ctrl.isText(), onchange: m.withAttr("value", ctrl.isText)}), + m("input", {type: "checkbox", value: ctrl.isText(), onchange: m.withAttr("value", ctrl.isText)}), "Text File" ]), m("p.help-block", "Select if the file to upload contains textual content.")@@ -87,5 +87,5 @@ ])
}; return u.modal(config); }; - -}());+ +}());
@@ -1,7 +1,7 @@
(function(){ 'use strict'; var app = window.LS || (window.LS = {}); - + app.widgets = {}; /* PANEL */@@ -11,7 +11,7 @@ var title = "";
var footer = ""; if (args.title){ title = m(".panel-heading", [ - m("h2.panel-title", [args.title]) + m("h2.panel-title", [args.title]) ]); } if (args.footer){@@ -26,7 +26,7 @@ footer
]); } }; - + /* PAGINATOR */ app.widgets.paginator = { view: function(ctrl, args){@@ -42,7 +42,7 @@ }
var first = (n === 0); var last = (n == max_page); var offset = args.limit * n; - sign = sign || n+1; + sign = sign || n+1; return m("li", {class: klass}, [m("a", { href: args.baseurl +(n+1), // assuming 10 elements per page //+"/"+obj.limit,@@ -51,7 +51,7 @@ }, [m.trust(sign)]
)] ); }; - + var pages = []; var prev; var next;@@ -78,7 +78,7 @@ pages.push(next);
return m("nav", [m("ul.pagination", pages)]); } }; - + /* DROPDOWN */ app.widgets.dropdown = { view: function(ctrl, args){@@ -90,9 +90,9 @@ }
return m(el, [ m("a.dropdown-toggle[href='#'][data-toggle='dropdown'][role='button'][aria-expanded='false']", [icon, m("span", " "+args.title+" "), m("span.caret")]), - m("ul.dropdown-menu[role='menu']", + m("ul.dropdown-menu[role='menu']", args.links.map(function(e){ - return m("li", + return m("li", [m("a", {href: e.path, config: m.route}, m.trust(e.title))]);})) ]); }@@ -102,18 +102,18 @@ /* DROPDOWN */
app.widgets.taglink = { view: function(ctrl, args) { var color = /^\$/.test(args.name) ? "warning" : "primary"; - return m("span.tag-label.label.label-"+color, + return m("span.tag-label.label.label-"+color, [m("i.fa.fa-tag"), " ", m("a", {href: "/tags/"+args.name, config:m.route}, args.name)]); } }; - + /* DOCLINK */ app.widgets.doclink = { view: function(ctrl, args) { return m("a", {href: "/document/view/"+args.id, config: m.route}, id); } }; - + /* TAGBUTTON */ app.widgets.tagbutton = { view: function(ctrl, args) {@@ -121,20 +121,20 @@ return m("a", {href: "/tags/"+args.name, config:m.route},
[m("i.fa.fa-tag"), " "+args.name+" ", m("span.badge", args.n)]); } }; - + /* TOOLBAR */ app.widgets.toolbar = { view: function(ctrl, args){ - return m("nav.toolbar.btn-group[role='group'][aria-label='...'].pull-right", + return m("nav.toolbar.btn-group[role='group'][aria-label='...'].pull-right", args.links.map(function(l){ - return m("a.btn.btn-default", - {onclick:l.action, config: l.config}, + return m("a.btn.btn-default", + {onclick:l.action, config: l.config}, [m("i.fa.fa-"+l.icon), " "+l.title]); - }) + }) ); } }; - + /* MODAL */ app.widgets.modal = { view: function(ctrl, args){@@ -144,13 +144,13 @@ }
if (!args.dismissText) { args.dismissText = "Close"; } - return m(".modal.fade", + return m(".modal.fade", {id: args.id, tabindex: "-1", role: "dialog"}, [ m(".modal-dialog", [ m(".modal-content", [ m(".modal-header", [ - m("button", {type: "button", class: "close", "data-dismiss": "modal"}, + m("button", {type: "button", class: "close", "data-dismiss": "modal"}, [m.trust("×")]), m("h4.modal-title", args.title) ]),@@ -166,4 +166,4 @@ );
} }; -}());+}());
@@ -7,44 +7,44 @@ var u = app.utils;
app.host = ""; //'http://localhost:9500'; var host = location.origin === app.host ? "" : app.host; - + Page.get = function(id) { var content = m.prop(""); return m.request({ - method: "GET", + method: "GET", url: host+"/docs/admin/md/"+id+".md", deserialize: function(value) { return value; } - }).then(function(content){ + }).then(function(content){ return u.markdown(content); }); }; - + Info.get = function(){ var content = m.prop(""); return m.request({ - method: "GET", + method: "GET", url: host+"/info" }).then(content); }; - + Doc.getByTag = function(tag, offset, limit) { offset = offset || 0; limit = limit || 10; var docs = m.prop(""); return m.request({ - method: "GET", + method: "GET", url: host+"/docs?contents=false&tags="+tag+"&limit="+limit+"&offset="+offset }).then(docs); }; - + Doc.search = function(search, offset, limit){ offset = offset || 0; limit = limit || 10; var docs = m.prop(""); return m.request({ - method: "GET", + method: "GET", url: host+"/docs?contents=false&search="+search+"&limit="+limit+"&offset="+offset, }).then(docs); };@@ -52,23 +52,23 @@
Doc.get = function(id) { var doc = m.prop(""); return m.request({ - method: "GET", + method: "GET", url: host+"/docs/"+id+"?raw=true" }).then(doc); }; - + Doc.delete = function(id){ return m.request({ - method: "DELETE", + method: "DELETE", url: host+"/docs/"+id }); }; - + Doc.put = function(doc, contentType){ xhrcfg = u.setContentType(doc, contentType); console.log("Doc.put - Saving Document:", doc); return m.request({ - method: "PUT", + method: "PUT", url: host+"/docs/"+doc.id, data: doc.data, serialize: function(data){@@ -77,7 +77,7 @@ },
config: xhrcfg }); }; - + Doc.upload = function(doc) { console.log("Doc.put - Uploading Document:", doc); return m.request({@@ -89,7 +89,7 @@ return data;
} }); }; - + Doc.patch = function(id, updatedTags){ return Doc.get(id).then(function(doc){ var tags = doc.tags;@@ -121,4 +121,4 @@ data: ops
}); }); }; -}());+}());
@@ -2,7 +2,7 @@ (function(){
'use strict'; var app = window.LS || (window.LS = {}); var u = app.utils = {}; - + // http://byronsalau.com/blog/how-to-create-a-guid-uuid-in-javascript/ u.guid = function(){ return 'xxxxxxxx-xxxx-4xxx-yxxx-xxxxxxxxxxxx'.replace(/[xy]/g, function(c) {@@ -10,7 +10,7 @@ var r = Math.random()*16|0, v = c === 'x' ? r : (r&0x3|0x8);
return v.toString(16); }); }; - + u.fixHeadings = function(html, maxheading){ var $content = $(html); var n = maxheading;@@ -26,7 +26,7 @@ }
return u.fixHeadings($content, maxheading); } }; - + u.setContentType = function(doc, contentType){ var type = ""; var subtype = "";@@ -80,7 +80,7 @@ var html = marked(s, {renderer: hs}).replace(/\{\{\$version\}\}/g, app.system.v);
var $html = $('<div>').append($(html).clone()); return u.fixHeadings($html, 2); }; - + /** * mod object: * @param {Object} mod@@ -129,7 +129,7 @@ u.date = function(date) {
return (date) ? new Date(Date.parse(date)).toUTCString() : "n/a"; }; - + u.showModal = function(sel){ return function(){ $(sel).modal();@@ -165,7 +165,7 @@ */
u.panel = function(obj){ return m.component(app.widgets.panel, obj); }; - + /** * @typedef {Object} PaginatorConfig * @prop {string} baseurl@@ -204,7 +204,7 @@ */
u.taglink = function(obj){ return m.component(app.widgets.taglink, obj); }; - + /** * Creates a DocLink component. * @param {Object} obj
@@ -64,4 +64,4 @@ * It is possible to upload local files instead of creating them by hand.
* Preview is available for images and HTML documents * Source code highlighting is available for Javascript, CSS, HTML, JSON and Markdown files. -![Document](images/app_document.png)+![Document](images/app_document.png)
@@ -551,4 +551,4 @@ Content-Length: 0
Access-Control-Allow-Headers: Content-Type Access-Control-Allow-Origin: * Server: LiteStore/1.0.3 -```+```
@@ -48,4 +48,4 @@ The [tags](class:kwd) table is used to store the associations between tags and documents. Tags can be added by users or add automatically by the system when a document is imported into the data store.
##### searchdata Table -This table is used as full-text index for searchable documents.+This table is used as full-text index for searchable documents.
@@ -7,4 +7,4 @@ * Andreas Rumpf and all the contributors to the [Nim Programming Language](http://nim-lang.org/), used to develop LiteStore.
* Leo Horie, for creating the [Mithril Javascript Framework](https://lhorie.github.io/mithril/), used to develop the LiteStore Administration App. * The creators and contributors to the [Bootstrap](http://getbootstrap.com/) CSS and Javascript framework, used by the LiteStore Administration App. * The creators and contributors to the [Ace Editor](http://ace.c9.io/), used by the LiteStore Administration App. -* Cristopher Jeffrey and all the contributors to the [Marked Javascript Library](https://github.com/chjj/marked) used by the LiteStore Administration App.+* Cristopher Jeffrey and all the contributors to the [Marked Javascript Library](https://github.com/chjj/marked) used by the LiteStore Administration App.
@@ -20,4 +20,4 @@ All system tags are prefixed by a [$](class:kwd) characters, and are used to identify the following document metadata. More specifically:
* **$dir:*directory*** – System tags starting with [$dir:](class:kwd) identify the name of a directory whose contents were imported into a data store. All files within the specified directory will be tagged with a [$dir:](class:kwd) system tag. Example: If a directory called **admin** is imported, imported files will be tagged with [$dir:admin](class:kwd). * **$type:*type*** – System tags starting with [$type:](class:kwd) identify the type of a document (i.e. the first portion of its content type). Example: Documents whose content type is **text/plain** will be tagged with [$type:text](class:kwd). -* **$subtype:*subtype*** – System tags starting with [$subtype:](class:kwd) identify the subtype of a document (i.e. the second portion of its content type). Example: Documents whose content type is **text/plain** will be tagged with [$subtype:plain](class:kwd).+* **$subtype:*subtype*** – System tags starting with [$subtype:](class:kwd) identify the subtype of a document (i.e. the second portion of its content type). Example: Documents whose content type is **text/plain** will be tagged with [$subtype:plain](class:kwd).
@@ -7,10 +7,7 @@ The easiest way to get LiteStore is by downloading one of the prebuilt binaries from the [Github Release Page][release]:
* [LiteStore for Mac OS X (x64)](https://github.com/h3rald/litestore/releases/download/{{$version}}litestore_{{$version}}_macosx_x64.zip) * [LiteStore for Windows (x64)](https://github.com/h3rald/litestore/releases/download/{{$version}}/litestore_{{$version}}_windows_x64.zip) - * [LiteStore for Windows (x86)](https://github.com/h3rald/litestore/releases/download/{{$version}}/litestore_{{$version}}_windows_x86.zip) * [LiteStore for Linux (x64)](https://github.com/h3rald/litestore/releases/download/{{$version}}/litestore_{{$version}}_linux_x64.zip) - * [LiteStore for Linux (x86)](https://github.com/h3rald/litestore/releases/download/{{$version}}/litestore_{{$version}}_linux_x86.zip) - * [LiteStore for Linux (ARM)](https://github.com/h3rald/litestore/releases/download/{{$version}}/litestore_{{$version}}_linux_arm.zip) ### Installing using Nimble
@@ -42,4 +42,4 @@ Your app could then be served on any desktop system able to run LiteStore (e.g. OSX, Windows, Linux, ...even on a [Raspberry Pi](https://www.raspberrypi.org)).
#### Static Site Backend -LiteStore can be configured to run in read-only mode, so that only GET, HEAD, or OPTIONS request are accepted by the server. This makes it ideal as a backend for static web site generated with something like [nanoc](http://nanoc.ws) or [Jekyll](http://jekyllrb.com).+LiteStore can be configured to run in read-only mode, so that only GET, HEAD, or OPTIONS request are accepted by the server. This makes it ideal as a backend for static web site generated with something like [nanoc](http://nanoc.ws) or [Jekyll](http://jekyllrb.com).
@@ -148,4 +148,4 @@
.search-result { margin: 5px 0; word-break: break-word; -}+}
@@ -8,5 +8,4 @@ font-size: inherit; // can't have font-size inherit on line above, so need to override
text-rendering: auto; // optimizelegibility throws things off #1094 -webkit-font-smoothing: antialiased; -moz-osx-font-smoothing: grayscale; - }
@@ -705,4 +705,3 @@ @fa-var-yen: "\f157";
@fa-var-youtube: "\f167"; @fa-var-youtube-play: "\f16a"; @fa-var-youtube-square: "\f166"; -
@@ -1,21 +1,12 @@
-import - litestorepkg/lib/x_sqlite3, - litestorepkg/lib/x_db_sqlite as db, - strutils, +import + strutils, os, - oids, - times, - json, - pegs, uri, - strtabs, - httpcore, - cgi, - base64 + httpcore import litestorepkg/lib/types, litestorepkg/lib/logger, - litestorepkg/lib/utils, + litestorepkg/lib/utils, litestorepkg/lib/core, litestorepkg/lib/cli, litestorepkg/lib/server@@ -28,7 +19,7 @@ utils
from asyncdispatch import runForever -{.compile: "litestorepkg/vendor/sqlite/libsqlite3.c".} +{.compile: "litestorepkg/vendor/sqlite/sqlite3.c".} {.passC: "-DSQLITE_ENABLE_FTS3=1 -DSQLITE_ENABLE_FTS3_PARENTHESIS -DSQLITE_ENABLE_JSON1".} proc executeOperation*() =@@ -37,7 +28,7 @@ let body = LS.execution.body
let ctype = LS.execution.ctype let uri = LS.execution.uri let operation = LS.execution.operation - var req:LSRequest + var req:LSRequest case operation.toUpperAscii: of "GET": req.reqMethod = HttpGet@@ -98,7 +89,7 @@ if LS.operation == opVacuum:
setup(false) vacuum LS.file else: - # Open Datastore + # Open Datastore setup(true) case LS.operation:@@ -127,13 +118,13 @@ for pair in pairs:
let data = pair.split("=") result[data[0]] = data[1] - proc query*(table: StringTableRef): string = + proc query*(table: StringTableRef): string = var params = newSeq[string](0) for key, value in pairs(table): params.add("$1=$2" % @[key, value]) return params.join("&") - proc newLSRequest(meth: HttpMethod, resource, id, body = "", params = newStringTable(), headers = newHttpHeaders()): LSRequest = + proc newLSRequest(meth: HttpMethod, resource, id, body = "", params = newStringTable(), headers = newHttpHeaders()): LSRequest = result.reqMethod = meth result.body = body result.headers = headers@@ -142,28 +133,28 @@
# Public API: Low-level proc getInfo*(): LSResponse = - return LS.getInfo() + return LS.getInfo(newLSRequest("info")) proc getRawDocuments*(options = newQueryOptions()): LSResponse = - return LS.getRawDocuments(options) + return LS.getRawDocuments(options, newLSRequest("docs")) proc getDocument*(id: string, options = newQueryOptions()): LSResponse = - return LS.getDocument(id, options) + return LS.getDocument(id, options, newLSRequest("docs", id)) proc getRawDocument*(id: string, options = newQueryOptions()): LSResponse = - return LS.getRawDocument(id, options) + return LS.getRawDocument(id, options, newLSRequest("docs", id)) proc deleteDocument*(id: string): LSResponse = - return LS.deleteDocument(id) + return LS.deleteDocument(id, newLSRequest("docs", id)) proc postDocument*(body, ct: string, folder=""): LSResponse = - return LS.postDocument(body, ct, folder) + return LS.postDocument(body, ct, folder, newLSRequest("docs", "", body)) proc putDocument*(id, body, ct: string): LSResponse = - return LS.putDocument(id, body, ct) + return LS.putDocument(id, body, ct newLSRequest("docs", id, body)) proc patchDocument*(id, body: string): LSResponse = - return LS.patchDocument(id, body) + return LS.patchDocument(id, body, newLSRequest("docs", id, body)) # Public API: High-level
@@ -1,27 +1,14 @@
define:release threads:on +# https://blog.filippo.io/easy-windows-and-linux-cross-compilers-for-macos/ + # https://gist.github.com/Drakulix/9881160 -amd64.windows.gcc.path = "/usr/local/mingw/bin" +amd64.windows.gcc.path = "/usr/local/bin" amd64.windows.gcc.exe = "x86_64-w64-mingw32-gcc" amd64.windows.gcc.linkerexe = "x86_64-w64-mingw32-gcc" -# https://gist.github.com/Drakulix/9881160 -i386.windows.gcc.path = "/usr/local/mingw/bin" -i386.windows.gcc.exe = "i686-w64-mingw32-gcc" -i386.windows.gcc.linkerexe = "i686-w64-mingw32-gcc" - # http://crossgcc.rts-software.org/doku.php?id=compiling_for_linux -i386.linux.gcc.path = "/usr/local/gcc-4.8.1-for-linux32/bin" -i386.linux.gcc.exe = "i586-pc-linux-gcc" -i386.linux.gcc.linkerexe = "i586-pc-linux-gcc" - -# http://crossgcc.rts-software.org/doku.php?id=compiling_for_linux -amd64.linux.gcc.path = "/usr/local/gcc-4.8.1-for-linux64/bin" -amd64.linux.gcc.exe = "x86_64-pc-linux-gcc" -amd64.linux.gcc.linkerexe = "x86_64-pc-linux-gcc" - -# http://www.jaredwolff.com/toolchains/ -arm.linux.gcc.path = "/usr/local/arm-none-linux-gnueabi/bin" -arm.linux.gcc.exe = "arm-none-linux-gnueabi-gcc" -arm.linux.gcc.linkerexe = "arm-none-linux-gnueabi-gcc" +amd64.linux.gcc.path = "/usr/local/bin" +amd64.linux.gcc.exe = "x86_64-linux-musl-gcc" +amd64.linux.gcc.linkerexe = "x86_64-linux-musl-gcc"
@@ -5,29 +5,29 @@
routes: # Just a simple, unrelated Jester route - get "/": + get "/": resp "Hello, World!" - + # Remapping LiteStore routes on Jester get re"^\/litestore\/(docs|info)\/?(.*)": let r = get(request.matches[0], request.matches[1], newStringTable(toSeq(request.params.pairs)), request.headers) - resp(r.code, r.content, r.headers["Content-Type"]) + resp(r.code, r.content, r.headers["Content-Type"]) post re"^\/litestore\/docs\/?(.*)": let r = post("docs", request.matches[0], request.body, request.headers) - resp(r.code, r.content, r.headers["Content-Type"]) + resp(r.code, r.content, r.headers["Content-Type"]) put re"^\/litestore\/docs\/?(.*)": let r = put("docs", request.matches[0], request.body, request.headers) - resp(r.code, r.content, r.headers["Content-Type"]) + resp(r.code, r.content, r.headers["Content-Type"]) patch re"^\/litestore\/docs\/?(.*)": let r = patch("docs", request.matches[0], request.body, request.headers) - resp(r.code, r.content, r.headers["Content-Type"]) + resp(r.code, r.content, r.headers["Content-Type"]) delete re"^\/litestore\/docs\/?(.*)": let r = delete("docs", request.matches[0], request.headers) - resp(r.code, r.content) + resp(r.code, r.content) head re"^\/litestore\/docs\/?(.*)": let r = head("docs", request.matches[0], request.headers)
@@ -1,6 +1,5 @@
-import +import asynchttpserver, - asyncdispatch, strutils, cgi, strtabs,@@ -8,7 +7,7 @@ pegs,
json, os, times -import +import types, contenttypes, core,@@ -188,10 +187,10 @@ content["datastore_version"] = %version
content["size"] = %($((LS.file.getFileSize().float/(1024*1024)).formatFloat(ffDecimal, 2)) & " MB") content["read_only"] = %LS.readonly content["log_level"] = %LS.loglevel - if LS.directory.len == 0: + if LS.directory.len == 0: content["directory"] = newJNull() - else: - content["directory"] = %LS.directory + else: + content["directory"] = %LS.directory content["mount"] = %LS.mount content["total_documents"] = %total_documents content["total_tags"] = %total_tags@@ -361,7 +360,7 @@ else:
discard # never happens really. -proc post(req: LSRequest, LS: LiteStore, resource: string, id = ""): LSResponse = +proc post(req: LSRequest, LS: LiteStore, resource: string, id = ""): LSResponse = if id == "": var ct = "text/plain" if req.headers.hasKey("Content-Type"):@@ -370,7 +369,7 @@ return LS.postDocument(req.body.strip, ct)
else: return resError(Http400, "Bad request: document ID cannot be specified in POST requests.") -proc put(req: LSRequest, LS: LiteStore, resource: string, id = ""): LSResponse = +proc put(req: LSRequest, LS: LiteStore, resource: string, id = ""): LSResponse = if id != "": var ct = "text/plain" if req.headers.hasKey("Content-Type"):@@ -379,13 +378,13 @@ return LS.putDocument(id, req.body.strip, ct)
else: return resError(Http400, "Bad request: document ID must be specified in PUT requests.") -proc delete(req: LSRequest, LS: LiteStore, resource: string, id = ""): LSResponse = +proc delete(req: LSRequest, LS: LiteStore, resource: string, id = ""): LSResponse = if id != "": return LS.deleteDocument(id) else: return resError(Http400, "Bad request: document ID must be specified in DELETE requests.") -proc patch(req: LSRequest, LS: LiteStore, resource: string, id = ""): LSResponse = +proc patch(req: LSRequest, LS: LiteStore, resource: string, id = ""): LSResponse = if id != "": return LS.patchDocument(id, req.body) else:@@ -415,24 +414,24 @@ return resError(Http500, "Unable to read file '$1'." % path)
else: return resError(Http404, "File '$1' not found." % path) else: - return resError(Http405, "Method not allowed: $1" % $req.reqMethod) + return resError(Http405, "Method not allowed: $1" % $req.reqMethod) -proc route*(req: LSRequest, LS: LiteStore, resource = "docs", id = ""): LSResponse = +proc route*(req: LSRequest, LS: LiteStore, resource = "docs", id = ""): LSResponse = var reqMethod = $req.reqMethod if req.headers.hasKey("X-HTTP-Method-Override"): reqMethod = req.headers["X-HTTP-Method-Override"] case reqMethod.toUpperAscii: of "POST": if LS.readonly: - return resError(Http405, "Method not allowed: $1" % $req.reqMethod) + return resError(Http405, "Method not allowed: $1" % $req.reqMethod) return validate(req, LS, resource, id, post) of "PUT": if LS.readonly: - return resError(Http405, "Method not allowed: $1" % $req.reqMethod) + return resError(Http405, "Method not allowed: $1" % $req.reqMethod) return validate(req, LS, resource, id, put) of "DELETE": if LS.readonly: - return resError(Http405, "Method not allowed: $1" % $req.reqMethod) + return resError(Http405, "Method not allowed: $1" % $req.reqMethod) return validate(req, LS, resource, id, delete) of "HEAD": return validate(req, LS, resource, id, head)@@ -442,7 +441,7 @@ of "GET":
return validate(req, LS, resource, id, get) of "PATCH": if LS.readonly: - return resError(Http405, "Method not allowed: $1" % $req.reqMethod) + return resError(Http405, "Method not allowed: $1" % $req.reqMethod) return validate(req, LS, resource, id, patch) else: - return resError(Http405, "Method not allowed: $1" % $req.reqMethod) + return resError(Http405, "Method not allowed: $1" % $req.reqMethod)
@@ -1,6 +1,5 @@
-import +import asynchttpserver, - asyncdispatch, strutils, cgi, strtabs,@@ -8,7 +7,7 @@ pegs,
json, os, times -import +import types, contenttypes, core,@@ -66,7 +65,7 @@ var fragments = querystring.split('&')
for f in fragments: f.parseQueryOption(options) -proc validate*(req: LSRequest, LS: LiteStore, resource: string, id: string, cb: proc(req: LSRequest, LS: LiteStore, resource: string, id: string):LSResponse): LSResponse = +proc validate*(req: LSRequest, LS: LiteStore, resource: string, id: string, cb: proc(req: LSRequest, LS: LiteStore, resource: string, id: string):LSResponse): LSResponse = if req.reqMethod == HttpPost or req.reqMethod == HttpPut or req.reqMethod == HttpPatch: var ct = "" let body = req.body.strip@@ -207,10 +206,10 @@ content["datastore_version"] = %version
content["size"] = %($((LS.file.getFileSize().float/(1024*1024)).formatFloat(ffDecimal, 2)) & " MB") content["read_only"] = %LS.readonly content["log_level"] = %LS.loglevel - if LS.directory.len == 0: + if LS.directory.len == 0: content["directory"] = newJNull() - else: - content["directory"] = %LS.directory + else: + content["directory"] = %LS.directory content["mount"] = %LS.mount content["total_documents"] = %total_documents content["total_tags"] = %total_tags@@ -221,7 +220,7 @@ result.code = Http200
proc postDocument*(LS: LiteStore, body: string, ct: string, folder=""): LSResponse = if not folder.isFolder: - return resError(Http400, "Invalid folder specified when creating document: $1" % folder) + return resError(Http400, "Invalid folder specified when creating document: $1" % folder) try: var doc = LS.store.createDocument(folder, body, ct) if doc != "":@@ -402,13 +401,13 @@ else:
discard # never happens really. -proc post*(req: LSRequest, LS: LiteStore, resource: string, id = ""): LSResponse = +proc post*(req: LSRequest, LS: LiteStore, resource: string, id = ""): LSResponse = var ct = "text/plain" if req.headers.hasKey("Content-Type"): ct = req.headers["Content-Type"] return LS.postDocument(req.body.strip, ct, id) -proc put*(req: LSRequest, LS: LiteStore, resource: string, id = ""): LSResponse = +proc put*(req: LSRequest, LS: LiteStore, resource: string, id = ""): LSResponse = if id != "": var ct = "text/plain" if req.headers.hasKey("Content-Type"):@@ -417,13 +416,13 @@ return LS.putDocument(id, req.body.strip, ct)
else: return resError(Http400, "Bad request: document ID must be specified in PUT requests.") -proc delete*(req: LSRequest, LS: LiteStore, resource: string, id = ""): LSResponse = +proc delete*(req: LSRequest, LS: LiteStore, resource: string, id = ""): LSResponse = if id != "": return LS.deleteDocument(id) else: return resError(Http400, "Bad request: document ID must be specified in DELETE requests.") -proc patch*(req: LSRequest, LS: LiteStore, resource: string, id = ""): LSResponse = +proc patch*(req: LSRequest, LS: LiteStore, resource: string, id = ""): LSResponse = if id != "": return LS.patchDocument(id, req.body) else:@@ -453,24 +452,24 @@ return resError(Http500, "Unable to read file '$1'." % path)
else: return resError(Http404, "File '$1' not found." % path) else: - return resError(Http405, "Method not allowed: $1" % $req.reqMethod) + return resError(Http405, "Method not allowed: $1" % $req.reqMethod) -proc route*(req: LSRequest, LS: LiteStore, resource = "docs", id = ""): LSResponse = +proc route*(req: LSRequest, LS: LiteStore, resource = "docs", id = ""): LSResponse = var reqMethod = $req.reqMethod if req.headers.hasKey("X-HTTP-Method-Override"): reqMethod = req.headers["X-HTTP-Method-Override"] case reqMethod.toUpperAscii: of "POST": if LS.readonly: - return resError(Http405, "Method not allowed: $1" % $req.reqMethod) + return resError(Http405, "Method not allowed: $1" % $req.reqMethod) return validate(req, LS, resource, id, post) of "PUT": if LS.readonly: - return resError(Http405, "Method not allowed: $1" % $req.reqMethod) + return resError(Http405, "Method not allowed: $1" % $req.reqMethod) return validate(req, LS, resource, id, put) of "DELETE": if LS.readonly: - return resError(Http405, "Method not allowed: $1" % $req.reqMethod) + return resError(Http405, "Method not allowed: $1" % $req.reqMethod) return validate(req, LS, resource, id, delete) of "HEAD": return validate(req, LS, resource, id, head)@@ -480,7 +479,7 @@ of "GET":
return validate(req, LS, resource, id, get) of "PATCH": if LS.readonly: - return resError(Http405, "Method not allowed: $1" % $req.reqMethod) + return resError(Http405, "Method not allowed: $1" % $req.reqMethod) return validate(req, LS, resource, id, patch) else: - return resError(Http405, "Method not allowed: $1" % $req.reqMethod) + return resError(Http405, "Method not allowed: $1" % $req.reqMethod)
@@ -1,6 +1,5 @@
-import +import asynchttpserver, - asyncdispatch, strutils, sequtils, cgi,@@ -9,7 +8,7 @@ pegs,
json, os, times -import +import types, contenttypes, core,@@ -51,7 +50,7 @@ else:
clauses.add("$1 ASC" % field) return clauses.join(", ") -proc selectClause*(str: string, options: var QueryOptions) = +proc selectClause*(str: string, options: var QueryOptions) = let tokens = """ path <- '$' (objItem / objField)+ ident <- [a-zA-Z0-9_]+@@ -61,10 +60,10 @@ objItem <- objField objIndex
""" let fields = peg(""" fields <- ^{field} (\s* ',' \s* {field})*$ - field <- path \s+ ('as' / 'AS') \s+ ident + field <- path \s+ ('as' / 'AS') \s+ ident """ & tokens) let field = peg(""" - field <- ^{path} \s+ ('as' / 'AS') \s+ {ident}$ + field <- ^{path} \s+ ('as' / 'AS') \s+ {ident}$ """ & tokens) var fieldMatches = newSeq[string](10) if str.strip.match(fields, fieldMatches):@@ -74,7 +73,7 @@ var rawTuple = newSeq[string](2)
if m.match(field, rawTuple): options.jsonSelect.add((path: rawTuple[0], alias: rawTuple[1])) -proc filterClauses*(str: string, options: var QueryOptions) = +proc filterClauses*(str: string, options: var QueryOptions) = let tokens = """ operator <- 'not eq' / 'eq' / 'gte' / 'gt' / 'lte' / 'lt' / 'contains' value <- string / number / 'null' / 'true' / 'false'@@ -181,7 +180,7 @@ var fragments = querystring.split('&')
for f in fragments: f.parseQueryOption(options) -proc validate*(req: LSRequest, LS: LiteStore, resource: string, id: string, cb: proc(req: LSRequest, LS: LiteStore, resource: string, id: string):LSResponse): LSResponse = +proc validate*(req: LSRequest, LS: LiteStore, resource: string, id: string, cb: proc(req: LSRequest, LS: LiteStore, resource: string, id: string):LSResponse): LSResponse = if req.reqMethod == HttpPost or req.reqMethod == HttpPut or req.reqMethod == HttpPatch: var ct = "" let body = req.body.strip@@ -252,11 +251,11 @@ d.elems[index] = value
case op: of "remove": d.elems.del(index) - of "add": + of "add": d.elems.insert(value, index) - of "replace": + of "replace": d.elems[index] = value - of "test": + of "test": if d.elems[index] != value: return false else:@@ -274,14 +273,14 @@ if d.hasKey(key):
d.delete(key) else: raise newException(EInvalidRequest, "key '$1' not found in path '$2'" % [key, path]) - of "add": + of "add": d[key] = value - of "replace": + of "replace": if d.hasKey(key): d[key] = value else: raise newException(EInvalidRequest, "key '$1' not found in path '$2'" % [key, path]) - of "test": + of "test": if dorig.hasKey(key): if dorig[key] != value: return false@@ -398,10 +397,10 @@ content["datastore_version"] = %version
content["size"] = %($((LS.file.getFileSize().float/(1024*1024)).formatFloat(ffDecimal, 2)) & " MB") content["read_only"] = %LS.readonly content["log_level"] = %LS.loglevel - if LS.directory.len == 0: + if LS.directory.len == 0: content["directory"] = newJNull() - else: - content["directory"] = %LS.directory + else: + content["directory"] = %LS.directory content["mount"] = %LS.mount content["total_documents"] = %total_documents content["total_tags"] = %total_tags@@ -412,7 +411,7 @@ result.code = Http200
proc postDocument*(LS: LiteStore, body: string, ct: string, folder=""): LSResponse = if not folder.isFolder: - return resError(Http400, "Invalid folder specified when creating document: $1" % folder) + return resError(Http400, "Invalid folder specified when creating document: $1" % folder) try: var doc = LS.store.createDocument(folder, body, ct) if doc != "":@@ -611,13 +610,13 @@ else:
discard # never happens really. -proc post*(req: LSRequest, LS: LiteStore, resource: string, id = ""): LSResponse = +proc post*(req: LSRequest, LS: LiteStore, resource: string, id = ""): LSResponse = var ct = "text/plain" if req.headers.hasKey("Content-Type"): ct = req.headers["Content-Type"] return LS.postDocument(req.body.strip, ct, id) -proc put*(req: LSRequest, LS: LiteStore, resource: string, id = ""): LSResponse = +proc put*(req: LSRequest, LS: LiteStore, resource: string, id = ""): LSResponse = if id != "": var ct = "text/plain" if req.headers.hasKey("Content-Type"):@@ -626,13 +625,13 @@ return LS.putDocument(id, req.body.strip, ct)
else: return resError(Http400, "Bad request: document ID must be specified in PUT requests.") -proc delete*(req: LSRequest, LS: LiteStore, resource: string, id = ""): LSResponse = +proc delete*(req: LSRequest, LS: LiteStore, resource: string, id = ""): LSResponse = if id != "": return LS.deleteDocument(id) else: return resError(Http400, "Bad request: document ID must be specified in DELETE requests.") -proc patch*(req: LSRequest, LS: LiteStore, resource: string, id = ""): LSResponse = +proc patch*(req: LSRequest, LS: LiteStore, resource: string, id = ""): LSResponse = if id != "": return LS.patchDocument(id, req.body) else:@@ -662,24 +661,24 @@ return resError(Http500, "Unable to read file '$1'." % path)
else: return resError(Http404, "File '$1' not found." % path) else: - return resError(Http405, "Method not allowed: $1" % $req.reqMethod) + return resError(Http405, "Method not allowed: $1" % $req.reqMethod) -proc route*(req: LSRequest, LS: LiteStore, resource = "docs", id = ""): LSResponse = +proc route*(req: LSRequest, LS: LiteStore, resource = "docs", id = ""): LSResponse = var reqMethod = $req.reqMethod if req.headers.hasKey("X-HTTP-Method-Override"): reqMethod = req.headers["X-HTTP-Method-Override"] case reqMethod.toUpperAscii: of "POST": if LS.readonly: - return resError(Http405, "Method not allowed: $1" % $req.reqMethod) + return resError(Http405, "Method not allowed: $1" % $req.reqMethod) return validate(req, LS, resource, id, post) of "PUT": if LS.readonly: - return resError(Http405, "Method not allowed: $1" % $req.reqMethod) + return resError(Http405, "Method not allowed: $1" % $req.reqMethod) return validate(req, LS, resource, id, put) of "DELETE": if LS.readonly: - return resError(Http405, "Method not allowed: $1" % $req.reqMethod) + return resError(Http405, "Method not allowed: $1" % $req.reqMethod) return validate(req, LS, resource, id, delete) of "HEAD": return validate(req, LS, resource, id, head)@@ -689,7 +688,7 @@ of "GET":
return validate(req, LS, resource, id, get) of "PATCH": if LS.readonly: - return resError(Http405, "Method not allowed: $1" % $req.reqMethod) + return resError(Http405, "Method not allowed: $1" % $req.reqMethod) return validate(req, LS, resource, id, patch) else: - return resError(Http405, "Method not allowed: $1" % $req.reqMethod) + return resError(Http405, "Method not allowed: $1" % $req.reqMethod)
@@ -1,6 +1,5 @@
-import +import asynchttpserver, - asyncdispatch, strutils, sequtils, cgi,@@ -9,7 +8,7 @@ pegs,
json, os, times -import +import types, contenttypes, core,@@ -51,7 +50,7 @@ else:
clauses.add("$1 ASC" % field) return clauses.join(", ") -proc selectClause*(str: string, options: var QueryOptions) = +proc selectClause*(str: string, options: var QueryOptions) = let tokens = """ path <- '$' (objItem / objField)+ ident <- [a-zA-Z0-9_]+@@ -61,10 +60,10 @@ objItem <- objField objIndex
""" let fields = peg(""" fields <- ^{field} (\s* ',' \s* {field})*$ - field <- path \s+ ('as' / 'AS') \s+ ident + field <- path \s+ ('as' / 'AS') \s+ ident """ & tokens) let field = peg(""" - field <- ^{path} \s+ ('as' / 'AS') \s+ {ident}$ + field <- ^{path} \s+ ('as' / 'AS') \s+ {ident}$ """ & tokens) var fieldMatches = newSeq[string](10) if str.strip.match(fields, fieldMatches):@@ -74,7 +73,7 @@ var rawTuple = newSeq[string](2)
if m.match(field, rawTuple): options.jsonSelect.add((path: rawTuple[0], alias: rawTuple[1])) -proc filterClauses*(str: string, options: var QueryOptions) = +proc filterClauses*(str: string, options: var QueryOptions) = let tokens = """ operator <- 'not eq' / 'eq' / 'gte' / 'gt' / 'lte' / 'lt' / 'contains' value <- string / number / 'null' / 'true' / 'false'@@ -205,7 +204,7 @@ var fragments = querystring.split('&')
for f in fragments: f.parseQueryOption(options) -proc validate*(req: LSRequest, LS: LiteStore, resource: string, id: string, cb: proc(req: LSRequest, LS: LiteStore, resource: string, id: string):LSResponse): LSResponse = +proc validate*(req: LSRequest, LS: LiteStore, resource: string, id: string, cb: proc(req: LSRequest, LS: LiteStore, resource: string, id: string):LSResponse): LSResponse = if req.reqMethod == HttpPost or req.reqMethod == HttpPut or req.reqMethod == HttpPatch: var ct = "" let body = req.body.strip@@ -275,11 +274,11 @@ d.elems[index] = value
case op: of "remove": d.elems.del(index) - of "add": + of "add": d.elems.insert(value, index) - of "replace": + of "replace": d.elems[index] = value - of "test": + of "test": if d.elems[index] != value: return false else:@@ -297,14 +296,14 @@ if d.hasKey(key):
d.delete(key) else: raise newException(EInvalidRequest, "key '$1' not found in path '$2'" % [key, path]) - of "add": + of "add": d[key] = value - of "replace": + of "replace": if d.hasKey(key): d[key] = value else: raise newException(EInvalidRequest, "key '$1' not found in path '$2'" % [key, path]) - of "test": + of "test": if dorig.hasKey(key): if dorig[key] != value: return false@@ -460,10 +459,10 @@ content["datastore_version"] = %version
content["size"] = %($((LS.file.getFileSize().float/(1024*1024)).formatFloat(ffDecimal, 2)) & " MB") content["read_only"] = %LS.readonly content["log_level"] = %LS.loglevel - if LS.directory.len == 0: + if LS.directory.len == 0: content["directory"] = newJNull() - else: - content["directory"] = %LS.directory + else: + content["directory"] = %LS.directory content["mount"] = %LS.mount content["total_documents"] = %total_documents content["total_tags"] = %total_tags@@ -475,9 +474,10 @@ result.code = Http200
proc postDocument*(LS: LiteStore, body: string, ct: string, folder="", req: LSRequest): LSResponse = if not folder.isFolder: - return resError(Http400, "Invalid folder specified when creating document: $1" % folder) + return resError(Http400, "Invalid folder specified when creating document: $1" % folder) try: var doc = LS.store.createDocument(folder, body, ct) + echo doc if doc != "": result.headers = ctJsonHeader() setOrigin(LS, req, result.headers)@@ -486,6 +486,7 @@ result.code = Http201
else: result = resError(Http500, "Unable to create document.") except: + eWarn() result = resError(Http500, "Unable to create document.") proc putDocument*(LS: LiteStore, id: string, body: string, ct: string, req: LSRequest): LSResponse =@@ -578,73 +579,73 @@
proc options*(req: LSRequest, LS: LiteStore, resource: string, id = ""): LSResponse = case resource: of "info": + result.headers = newHttpHeaders(TAB_HEADERS) + setOrigin(LS, req, result.headers) + result.headers["Allow"] = "GET, OPTIONS" + result.headers["Access-Control-Allow-Methods"] = "GET, OPTIONS" if id != "": return resError(Http404, "Info '$1' not found." % id) else: - result.code = Http200 + result.code = Http204 result.content = "" - result.headers = newHttpHeaders(TAB_HEADERS) - setOrigin(LS, req, result.headers) - result.headers["Allow"] = "GET,OPTIONS" - result.headers["Access-Control-Allow-Methods"] = "GET,OPTIONS" of "dir": - result.code = Http200 + result.code = Http204 result.content = "" result.headers = newHttpHeaders(TAB_HEADERS) setOrigin(LS, req, result.headers) - result.headers["Allow"] = "GET,OPTIONS" - result.headers["Access-Control-Allow-Methods"] = "GET,OPTIONS" + result.headers["Allow"] = "GET, OPTIONS" + result.headers["Access-Control-Allow-Methods"] = "GET, OPTIONS" of "tags": - result.code = Http200 + result.code = Http204 result.content = "" result.headers = newHttpHeaders(TAB_HEADERS) setOrigin(LS, req, result.headers) - result.headers["Allow"] = "GET,OPTIONS" - result.headers["Access-Control-Allow-Methods"] = "GET,OPTIONS" + result.headers["Allow"] = "GET, OPTIONS" + result.headers["Access-Control-Allow-Methods"] = "GET, OPTIONS" of "docs": var folder: string if id.isFolder: folder = id if folder.len > 0: - result.code = Http200 + result.code = Http204 result.content = "" if LS.readonly: result.headers = newHttpHeaders(TAB_HEADERS) setOrigin(LS, req, result.headers) - result.headers["Allow"] = "HEAD,GET,OPTIONS" - result.headers["Access-Control-Allow-Methods"] = "HEAD,GET,OPTIONS" + result.headers["Allow"] = "HEAD, GET, OPTIONS" + result.headers["Access-Control-Allow-Methods"] = "HEAD, GET, OPTIONS" else: result.headers = newHttpHeaders(TAB_HEADERS) setOrigin(LS, req, result.headers) - result.headers["Allow"] = "HEAD,GET,OPTIONS,POST,PUT" - result.headers["Access-Control-Allow-Methods"] = "HEAD,GET,OPTIONS,POST,PUT" + result.headers["Allow"] = "HEAD, GET, OPTIONS, POST, PUT" + result.headers["Access-Control-Allow-Methods"] = "HEAD, GET, OPTIONS, POST, PUT" elif id != "": - result.code = Http200 + result.code = Http204 result.content = "" if LS.readonly: result.headers = newHttpHeaders(TAB_HEADERS) setOrigin(LS, req, result.headers) - result.headers["Allow"] = "HEAD,GET,OPTIONS" - result.headers["Access-Control-Allow-Methods"] = "HEAD,GET,OPTIONS" + result.headers["Allow"] = "HEAD, GET, OPTIONS" + result.headers["Access-Control-Allow-Methods"] = "HEAD, GET, OPTIONS" else: result.headers = newHttpHeaders(TAB_HEADERS) setOrigin(LS, req, result.headers) - result.headers["Allow"] = "HEAD,GET,OPTIONS,PUT,PATCH,DELETE" + result.headers["Allow"] = "HEAD, GET, OPTIONS, PUT, PATCH, DELETE" result.headers["Allow-Patch"] = "application/json-patch+json" - result.headers["Access-Control-Allow-Methods"] = "HEAD,GET,OPTIONS,PUT,PATCH,DELETE" + result.headers["Access-Control-Allow-Methods"] = "HEAD, GET, OPTIONS, PUT, PATCH, DELETE" else: - result.code = Http200 + result.code = Http204 result.content = "" if LS.readonly: result.headers = newHttpHeaders(TAB_HEADERS) setOrigin(LS, req, result.headers) - result.headers["Allow"] = "HEAD,GET,OPTIONS" - result.headers["Access-Control-Allow-Methods"] = "HEAD,GET,OPTIONS" + result.headers["Allow"] = "HEAD, GET, OPTIONS" + result.headers["Access-Control-Allow-Methods"] = "HEAD, GET, OPTIONS" else: result.headers = newHttpHeaders(TAB_HEADERS) setOrigin(LS, req, result.headers) - result.headers["Allow"] = "HEAD,GET,OPTIONS,POST" - result.headers["Access-Control-Allow-Methods"] = "HEAD,GET,OPTIONS,POST" + result.headers["Allow"] = "HEAD, GET, OPTIONS, POST" + result.headers["Access-Control-Allow-Methods"] = "HEAD, GET, OPTIONS, POST" else: discard # never happens really.@@ -683,12 +684,15 @@ return LS.getDocument(id, options, req)
else: return LS.getRawDocuments(options, req) except: + let e = getCurrentException() + let trace = e.getStackTrace() + echo trace return resError(Http400, "Bad Request - $1" % getCurrentExceptionMsg()) of "tags": var options = newQueryOptions() try: parseQueryOptions(req.url.query, options); - if id != "": + if id != "": return LS.getTag(id, options, req) else: return LS.getTags(options, req)@@ -701,13 +705,13 @@ return LS.getInfo(req)
else: discard # never happens really. -proc post*(req: LSRequest, LS: LiteStore, resource: string, id = ""): LSResponse = +proc post*(req: LSRequest, LS: LiteStore, resource: string, id = ""): LSResponse = var ct = "text/plain" if req.headers.hasKey("Content-Type"): ct = req.headers["Content-Type"] return LS.postDocument(req.body.strip, ct, id, req) -proc put*(req: LSRequest, LS: LiteStore, resource: string, id = ""): LSResponse = +proc put*(req: LSRequest, LS: LiteStore, resource: string, id = ""): LSResponse = if id != "": var ct = "text/plain" if req.headers.hasKey("Content-Type"):@@ -716,13 +720,13 @@ return LS.putDocument(id, req.body.strip, ct, req)
else: return resError(Http400, "Bad request: document ID must be specified in PUT requests.") -proc delete*(req: LSRequest, LS: LiteStore, resource: string, id = ""): LSResponse = +proc delete*(req: LSRequest, LS: LiteStore, resource: string, id = ""): LSResponse = if id != "": return LS.deleteDocument(id, req) else: return resError(Http400, "Bad request: document ID must be specified in DELETE requests.") -proc patch*(req: LSRequest, LS: LiteStore, resource: string, id = ""): LSResponse = +proc patch*(req: LSRequest, LS: LiteStore, resource: string, id = ""): LSResponse = if id != "": return LS.patchDocument(id, req.body, req) else:@@ -753,24 +757,24 @@ return resError(Http500, "Unable to read file '$1'." % path)
else: return resError(Http404, "File '$1' not found." % path) else: - return resError(Http405, "Method not allowed: $1" % $req.reqMethod) + return resError(Http405, "Method not allowed: $1" % $req.reqMethod) -proc route*(req: LSRequest, LS: LiteStore, resource = "docs", id = ""): LSResponse = +proc route*(req: LSRequest, LS: LiteStore, resource = "docs", id = ""): LSResponse = var reqMethod = $req.reqMethod if req.headers.hasKey("X-HTTP-Method-Override"): reqMethod = req.headers["X-HTTP-Method-Override"] case reqMethod.toUpperAscii: of "POST": if LS.readonly: - return resError(Http405, "Method not allowed: $1" % $req.reqMethod) + return resError(Http405, "Method not allowed: $1" % $req.reqMethod) return validate(req, LS, resource, id, post) of "PUT": if LS.readonly: - return resError(Http405, "Method not allowed: $1" % $req.reqMethod) + return resError(Http405, "Method not allowed: $1" % $req.reqMethod) return validate(req, LS, resource, id, put) of "DELETE": if LS.readonly: - return resError(Http405, "Method not allowed: $1" % $req.reqMethod) + return resError(Http405, "Method not allowed: $1" % $req.reqMethod) return validate(req, LS, resource, id, delete) of "HEAD": return validate(req, LS, resource, id, head)@@ -780,7 +784,7 @@ of "GET":
return validate(req, LS, resource, id, get) of "PATCH": if LS.readonly: - return resError(Http405, "Method not allowed: $1" % $req.reqMethod) + return resError(Http405, "Method not allowed: $1" % $req.reqMethod) return validate(req, LS, resource, id, patch) else: - return resError(Http405, "Method not allowed: $1" % $req.reqMethod) + return resError(Http405, "Method not allowed: $1" % $req.reqMethod)
@@ -1,8 +1,6 @@
import parseopt, - strutils, - strtabs, - json + strutils import logger, config,@@ -11,7 +9,7 @@ utils
const favicon = "../../admin/favicon.ico".slurp -var +var operation = opRun directory:string = "" readonly = false@@ -23,10 +21,10 @@ exFile:string = ""
exBody:string = "" exType:string = "" exUri:string = "" - + let usage* = appname & " v" & pkgVersion & " - Lightweight REST Document Store" & """ - + (c) 2015-2018 Fabio Cevasco Usage:
@@ -1,6 +1,6 @@
const pkgName* = "litestore" - pkgVersion* = "1.5.0" + pkgVersion* = "1.5.1" pkgAuthor* = "Fabio Cevasco" pkgDescription* = "Self-contained, lightweight, RESTful document store." pkgLicense* = "MIT"@@ -10,4 +10,3 @@ var
file* = "data.db" address* = "127.0.0.1" port* = 9500 -
@@ -1,2468 +1,2468 @@
-[ - { +[ + { ".123":"application\/vnd.lotus-1-2-3" }, - { + { ".3dml":"text\/vnd.in3d.3dml" }, - { + { ".3g2":"video\/3gpp2" }, - { + { ".3gp":"video\/3gpp" }, - { + { ".a":"application\/octet-stream" }, - { + { ".aab":"application\/x-authorware-bin" }, - { + { ".aac":"audio\/x-aac" }, - { + { ".aam":"application\/x-authorware-map" }, - { + { ".aas":"application\/x-authorware-seg" }, - { + { ".abw":"application\/x-abiword" }, - { + { ".acc":"application\/vnd.americandynamics.acc" }, - { + { ".ace":"application\/x-ace-compressed" }, - { + { ".acu":"application\/vnd.acucobol" }, - { + { ".acutc":"application\/vnd.acucorp" }, - { + { ".adp":"audio\/adpcm" }, - { + { ".aep":"application\/vnd.audiograph" }, - { + { ".afm":"application\/x-font-type1" }, - { + { ".afp":"application\/vnd.ibm.modcap" }, - { + { ".ai":"application\/postscript" }, - { + { ".aif":"audio\/x-aiff" }, - { + { ".aifc":"audio\/x-aiff" }, - { + { ".aiff":"audio\/x-aiff" }, - { + { ".air":"application\/vnd.adobe.air-application-installer-package+zip" }, - { + { ".ami":"application\/vnd.amiga.ami" }, - { + { ".apk":"application\/vnd.android.package-archive" }, - { + { ".application":"application\/x-ms-application" }, - { + { ".apr":"application\/vnd.lotus-approach" }, - { + { ".asc":"application\/pgp-signature" }, - { + { ".asf":"video\/x-ms-asf" }, - { + { ".asm":"text\/x-asm" }, - { + { ".aso":"application\/vnd.accpac.simply.aso" }, - { + { ".asx":"video\/x-ms-asf" }, - { + { ".atc":"application\/vnd.acucorp" }, - { + { ".atom":"application\/atom+xml" }, - { + { ".atomcat":"application\/atomcat+xml" }, - { + { ".atomsvc":"application\/atomsvc+xml" }, - { + { ".atx":"application\/vnd.antix.game-component" }, - { + { ".au":"audio\/basic" }, - { + { ".avi":"video\/x-msvideo" }, - { + { ".aw":"application\/applixware" }, - { + { ".azf":"application\/vnd.airzip.filesecure.azf" }, - { + { ".azs":"application\/vnd.airzip.filesecure.azs" }, - { + { ".azw":"application\/vnd.amazon.ebook" }, - { + { ".bat":"application\/x-msdownload" }, - { + { ".bcpio":"application\/x-bcpio" }, - { + { ".bdf":"application\/x-font-bdf" }, - { + { ".bdm":"application\/vnd.syncml.dm+wbxml" }, - { + { ".bh2":"application\/vnd.fujitsu.oasysprs" }, - { + { ".bin":"application\/octet-stream" }, - { + { ".bmi":"application\/vnd.bmi" }, - { + { ".bmp":"image\/bmp" }, - { + { ".book":"application\/vnd.framemaker" }, - { + { ".box":"application\/vnd.previewsystems.box" }, - { + { ".boz":"application\/x-bzip2" }, - { + { ".bpk":"application\/octet-stream" }, - { + { ".btif":"image\/prs.btif" }, - { + { ".bz":"application\/x-bzip" }, - { + { ".bz2":"application\/x-bzip2" }, - { + { ".c":"text\/x-c" }, - { + { ".c4d":"application\/vnd.clonk.c4group" }, - { + { ".c4f":"application\/vnd.clonk.c4group" }, - { + { ".c4g":"application\/vnd.clonk.c4group" }, - { + { ".c4p":"application\/vnd.clonk.c4group" }, - { + { ".c4u":"application\/vnd.clonk.c4group" }, - { + { ".cab":"application\/vnd.ms-cab-compressed" }, - { + { ".car":"application\/vnd.curl.car" }, - { + { ".cat":"application\/vnd.ms-pki.seccat" }, - { + { ".cc":"text\/x-c" }, - { + { ".cct":"application\/x-director" }, - { + { ".ccxml":"application\/ccxml+xml" }, - { + { ".cdbcmsg":"application\/vnd.contact.cmsg" }, - { + { ".cdf":"application\/x-netcdf" }, - { + { ".cdkey":"application\/vnd.mediastation.cdkey" }, - { + { ".cdx":"chemical\/x-cdx" }, - { + { ".cdxml":"application\/vnd.chemdraw+xml" }, - { + { ".cdy":"application\/vnd.cinderella" }, - { + { ".cer":"application\/pkix-cert" }, - { + { ".cgm":"image\/cgm" }, - { + { ".chat":"application\/x-chat" }, - { + { ".chm":"application\/vnd.ms-htmlhelp" }, - { + { ".chrt":"application\/vnd.kde.kchart" }, - { + { ".cif":"chemical\/x-cif" }, - { + { ".cii":"application\/vnd.anser-web-certificate-issue-initiation" }, - { + { ".cil":"application\/vnd.ms-artgalry" }, - { + { ".cla":"application\/vnd.claymore" }, - { + { ".class":"application\/java-vm" }, - { + { ".clkk":"application\/vnd.crick.clicker.keyboard" }, - { + { ".clkp":"application\/vnd.crick.clicker.palette" }, - { + { ".clkt":"application\/vnd.crick.clicker.template" }, - { + { ".clkw":"application\/vnd.crick.clicker.wordbank" }, - { + { ".clkx":"application\/vnd.crick.clicker" }, - { + { ".clp":"application\/x-msclip" }, - { + { ".cmc":"application\/vnd.cosmocaller" }, - { + { ".cmdf":"chemical\/x-cmdf" }, - { + { ".cml":"chemical\/x-cml" }, - { + { ".cmp":"application\/vnd.yellowriver-custom-menu" }, - { + { ".cmx":"image\/x-cmx" }, - { + { ".cod":"application\/vnd.rim.cod" }, - { + { ".com":"application\/x-msdownload" }, - { + { ".conf":"text\/plain" }, - { + { ".cpio":"application\/x-cpio" }, - { + { ".cpp":"text\/x-c" }, - { + { ".cpt":"application\/mac-compactpro" }, - { + { ".crd":"application\/x-mscardfile" }, - { + { ".crl":"application\/pkix-crl" }, - { + { ".crt":"application\/x-x509-ca-cert" }, - { + { ".csh":"application\/x-csh" }, - { + { ".csml":"chemical\/x-csml" }, - { + { ".csp":"application\/vnd.commonspace" }, - { + { ".css":"text\/css" }, - { + { ".cst":"application\/x-director" }, - { + { ".csv":"text\/csv" }, - { + { ".cu":"application\/cu-seeme" }, - { + { ".curl":"text\/vnd.curl" }, - { + { ".cww":"application\/prs.cww" }, - { + { ".cxt":"application\/x-director" }, - { + { ".cxx":"text\/x-c" }, - { + { ".daf":"application\/vnd.mobius.daf" }, - { + { ".dataless":"application\/vnd.fdsn.seed" }, - { + { ".davmount":"application\/davmount+xml" }, - { + { ".dcr":"application\/x-director" }, - { + { ".dcurl":"text\/vnd.curl.dcurl" }, - { + { ".dd2":"application\/vnd.oma.dd2+xml" }, - { + { ".ddd":"application\/vnd.fujixerox.ddd" }, - { + { ".deb":"application\/x-debian-package" }, - { + { ".def":"text\/plain" }, - { + { ".deploy":"application\/octet-stream" }, - { + { ".der":"application\/x-x509-ca-cert" }, - { + { ".dfac":"application\/vnd.dreamfactory" }, - { + { ".dic":"text\/x-c" }, - { + { ".diff":"text\/plain" }, - { + { ".dir":"application\/x-director" }, - { + { ".dis":"application\/vnd.mobius.dis" }, - { + { ".dist":"application\/octet-stream" }, - { + { ".distz":"application\/octet-stream" }, - { + { ".djv":"image\/vnd.djvu" }, - { + { ".djvu":"image\/vnd.djvu" }, - { + { ".dll":"application\/x-msdownload" }, - { + { ".dmg":"application\/octet-stream" }, - { + { ".dms":"application\/octet-stream" }, - { + { ".dna":"application\/vnd.dna" }, - { + { ".doc":"application\/msword" }, - { + { ".docm":"application\/vnd.ms-word.document.macroenabled.12" }, - { + { ".docx":"application\/vnd.openxmlformats-officedocument.wordprocessingml.document" }, - { + { ".dot":"application\/msword" }, - { + { ".dotm":"application\/vnd.ms-word.template.macroenabled.12" }, - { + { ".dotx":"application\/vnd.openxmlformats-officedocument.wordprocessingml.template" }, - { + { ".dp":"application\/vnd.osgi.dp" }, - { + { ".dpg":"application\/vnd.dpgraph" }, - { + { ".dsc":"text\/prs.lines.tag" }, - { + { ".dtb":"application\/x-dtbook+xml" }, - { + { ".dtd":"application\/xml-dtd" }, - { + { ".dts":"audio\/vnd.dts" }, - { + { ".dtshd":"audio\/vnd.dts.hd" }, - { + { ".dump":"application\/octet-stream" }, - { + { ".dvi":"application\/x-dvi" }, - { + { ".dwf":"model\/vnd.dwf" }, - { + { ".dwg":"image\/vnd.dwg" }, - { + { ".dxf":"image\/vnd.dxf" }, - { + { ".dxp":"application\/vnd.spotfire.dxp" }, - { + { ".dxr":"application\/x-director" }, - { + { ".ecelp4800":"audio\/vnd.nuera.ecelp4800" }, - { + { ".ecelp7470":"audio\/vnd.nuera.ecelp7470" }, - { + { ".ecelp9600":"audio\/vnd.nuera.ecelp9600" }, - { + { ".ecma":"application\/ecmascript" }, - { + { ".edm":"application\/vnd.novadigm.edm" }, - { + { ".edx":"application\/vnd.novadigm.edx" }, - { + { ".efif":"application\/vnd.picsel" }, - { + { ".ei6":"application\/vnd.pg.osasli" }, - { + { ".elc":"application\/octet-stream" }, - { + { ".eml":"message\/rfc822" }, - { + { ".emma":"application\/emma+xml" }, - { + { ".eol":"audio\/vnd.digital-winds" }, - { + { ".eot":"application\/vnd.ms-fontobject" }, - { + { ".eps":"application\/postscript" }, - { + { ".epub":"application\/epub+zip" }, - { + { ".es3":"application\/vnd.eszigno3+xml" }, - { + { ".esf":"application\/vnd.epson.esf" }, - { + { ".et3":"application\/vnd.eszigno3+xml" }, - { + { ".etx":"text\/x-setext" }, - { + { ".exe":"application\/x-msdownload" }, - { + { ".ext":"application\/vnd.novadigm.ext" }, - { + { ".ez":"application\/andrew-inset" }, - { + { ".ez2":"application\/vnd.ezpix-album" }, - { + { ".ez3":"application\/vnd.ezpix-package" }, - { + { ".f":"text\/x-fortran" }, - { + { ".f4v":"video\/x-f4v" }, - { + { ".f77":"text\/x-fortran" }, - { + { ".f90":"text\/x-fortran" }, - { + { ".fbs":"image\/vnd.fastbidsheet" }, - { + { ".fdf":"application\/vnd.fdf" }, - { + { ".fe_launch":"application\/vnd.denovo.fcselayout-link" }, - { + { ".fg5":"application\/vnd.fujitsu.oasysgp" }, - { + { ".fgd":"application\/x-director" }, - { + { ".fh":"image\/x-freehand" }, - { + { ".fh4":"image\/x-freehand" }, - { + { ".fh5":"image\/x-freehand" }, - { + { ".fh7":"image\/x-freehand" }, - { + { ".fhc":"image\/x-freehand" }, - { + { ".fig":"application\/x-xfig" }, - { + { ".fli":"video\/x-fli" }, - { + { ".flo":"application\/vnd.micrografx.flo" }, - { + { ".flv":"video\/x-flv" }, - { + { ".flw":"application\/vnd.kde.kivio" }, - { + { ".flx":"text\/vnd.fmi.flexstor" }, - { + { ".fly":"text\/vnd.fly" }, - { + { ".fm":"application\/vnd.framemaker" }, - { + { ".fnc":"application\/vnd.frogans.fnc" }, - { + { ".for":"text\/x-fortran" }, - { + { ".fpx":"image\/vnd.fpx" }, - { + { ".frame":"application\/vnd.framemaker" }, - { + { ".fsc":"application\/vnd.fsc.weblaunch" }, - { + { ".fst":"image\/vnd.fst" }, - { + { ".ftc":"application\/vnd.fluxtime.clip" }, - { + { ".fti":"application\/vnd.anser-web-funds-transfer-initiation" }, - { + { ".fvt":"video\/vnd.fvt" }, - { + { ".fzs":"application\/vnd.fuzzysheet" }, - { + { ".g3":"image\/g3fax" }, - { + { ".gac":"application\/vnd.groove-account" }, - { + { ".gdl":"model\/vnd.gdl" }, - { + { ".geo":"application\/vnd.dynageo" }, - { + { ".gex":"application\/vnd.geometry-explorer" }, - { + { ".ggb":"application\/vnd.geogebra.file" }, - { + { ".ggt":"application\/vnd.geogebra.tool" }, - { + { ".ghf":"application\/vnd.groove-help" }, - { + { ".gif":"image\/gif" }, - { + { ".gim":"application\/vnd.groove-identity-message" }, - { + { ".gmx":"application\/vnd.gmx" }, - { + { ".gnumeric":"application\/x-gnumeric" }, - { + { ".gph":"application\/vnd.flographit" }, - { + { ".gqf":"application\/vnd.grafeq" }, - { + { ".gqs":"application\/vnd.grafeq" }, - { + { ".gram":"application\/srgs" }, - { + { ".gre":"application\/vnd.geometry-explorer" }, - { + { ".grv":"application\/vnd.groove-injector" }, - { + { ".grxml":"application\/srgs+xml" }, - { + { ".gsf":"application\/x-font-ghostscript" }, - { + { ".gtar":"application\/x-gtar" }, - { + { ".gtm":"application\/vnd.groove-tool-message" }, - { + { ".gtw":"model\/vnd.gtw" }, - { + { ".gv":"text\/vnd.graphviz" }, - { + { ".gz":"application\/x-gzip" }, - { + { ".h":"text\/x-c" }, - { + { ".h261":"video\/h261" }, - { + { ".h263":"video\/h263" }, - { + { ".h264":"video\/h264" }, - { + { ".hbci":"application\/vnd.hbci" }, - { + { ".hdf":"application\/x-hdf" }, - { + { ".hh":"text\/x-c" }, - { + { ".hlp":"application\/winhlp" }, - { + { ".hpgl":"application\/vnd.hp-hpgl" }, - { + { ".hpid":"application\/vnd.hp-hpid" }, - { + { ".hps":"application\/vnd.hp-hps" }, - { + { ".hqx":"application\/mac-binhex40" }, - { + { ".htke":"application\/vnd.kenameaapp" }, - { + { ".htm":"text\/html" }, - { + { ".html":"text\/html" }, - { + { ".hvd":"application\/vnd.yamaha.hv-dic" }, - { + { ".hvp":"application\/vnd.yamaha.hv-voice" }, - { + { ".hvs":"application\/vnd.yamaha.hv-script" }, - { + { ".icc":"application\/vnd.iccprofile" }, - { + { ".ice":"x-conference\/x-cooltalk" }, - { + { ".icm":"application\/vnd.iccprofile" }, - { + { ".ico":"image\/x-icon" }, - { + { ".ics":"text\/calendar" }, - { + { ".ief":"image\/ief" }, - { + { ".ifb":"text\/calendar" }, - { + { ".ifm":"application\/vnd.shana.informed.formdata" }, - { + { ".iges":"model\/iges" }, - { + { ".igl":"application\/vnd.igloader" }, - { + { ".igs":"model\/iges" }, - { + { ".igx":"application\/vnd.micrografx.igx" }, - { + { ".iif":"application\/vnd.shana.informed.interchange" }, - { + { ".imp":"application\/vnd.accpac.simply.imp" }, - { + { ".ims":"application\/vnd.ms-ims" }, - { + { ".in":"text\/plain" }, - { + { ".ipk":"application\/vnd.shana.informed.package" }, - { + { ".irm":"application\/vnd.ibm.rights-management" }, - { + { ".irp":"application\/vnd.irepository.package+xml" }, - { + { ".iso":"application\/octet-stream" }, - { + { ".itp":"application\/vnd.shana.informed.formtemplate" }, - { + { ".ivp":"application\/vnd.immervision-ivp" }, - { + { ".ivu":"application\/vnd.immervision-ivu" }, - { + { ".jad":"text\/vnd.sun.j2me.app-descriptor" }, - { + { ".jam":"application\/vnd.jam" }, - { + { ".jar":"application\/java-archive" }, - { + { ".java":"text\/x-java-source" }, - { + { ".jisp":"application\/vnd.jisp" }, - { + { ".jlt":"application\/vnd.hp-jlyt" }, - { + { ".jnlp":"application\/x-java-jnlp-file" }, - { + { ".joda":"application\/vnd.joost.joda-archive" }, - { + { ".jpe":"image\/jpeg" }, - { + { ".jpeg":"image\/jpeg" }, - { + { ".jpg":"image\/jpeg" }, - { + { ".jpgm":"video\/jpm" }, - { + { ".jpgv":"video\/jpeg" }, - { + { ".jpm":"video\/jpm" }, - { + { ".js":"application\/javascript" }, - { + { ".json":"application\/json" }, - { + { ".kar":"audio\/midi" }, - { + { ".karbon":"application\/vnd.kde.karbon" }, - { + { ".kfo":"application\/vnd.kde.kformula" }, - { + { ".kia":"application\/vnd.kidspiration" }, - { + { ".kil":"application\/x-killustrator" }, - { + { ".kml":"application\/vnd.google-earth.kml+xml" }, - { + { ".kmz":"application\/vnd.google-earth.kmz" }, - { + { ".kne":"application\/vnd.kinar" }, - { + { ".knp":"application\/vnd.kinar" }, - { + { ".kon":"application\/vnd.kde.kontour" }, - { + { ".kpr":"application\/vnd.kde.kpresenter" }, - { + { ".kpt":"application\/vnd.kde.kpresenter" }, - { + { ".ksh":"text\/plain" }, - { + { ".ksp":"application\/vnd.kde.kspread" }, - { + { ".ktr":"application\/vnd.kahootz" }, - { + { ".ktz":"application\/vnd.kahootz" }, - { + { ".kwd":"application\/vnd.kde.kword" }, - { + { ".kwt":"application\/vnd.kde.kword" }, - { + { ".latex":"application\/x-latex" }, - { + { ".lbd":"application\/vnd.llamagraphics.life-balance.desktop" }, - { + { ".lbe":"application\/vnd.llamagraphics.life-balance.exchange+xml" }, - { + { ".les":"application\/vnd.hhe.lesson-player" }, { ".less":"text/x-less" }, - { + { ".lha":"application\/octet-stream" }, - { + { ".link66":"application\/vnd.route66.link66+xml" }, - { + { ".list":"text\/plain" }, - { + { ".list3820":"application\/vnd.ibm.modcap" }, - { + { ".listafp":"application\/vnd.ibm.modcap" }, - { + { ".log":"text\/plain" }, - { + { ".lostxml":"application\/lost+xml" }, - { + { ".lrf":"application\/octet-stream" }, - { + { ".lrm":"application\/vnd.ms-lrm" }, - { + { ".ltf":"application\/vnd.frogans.ltf" }, - { + { ".lvp":"audio\/vnd.lucent.voice" }, - { + { ".lwp":"application\/vnd.lotus-wordpro" }, - { + { ".lzh":"application\/octet-stream" }, - { + { ".m13":"application\/x-msmediaview" }, - { + { ".m14":"application\/x-msmediaview" }, - { + { ".m1v":"video\/mpeg" }, - { + { ".m2a":"audio\/mpeg" }, - { + { ".m2v":"video\/mpeg" }, - { + { ".m3a":"audio\/mpeg" }, - { + { ".m3u":"audio\/x-mpegurl" }, - { + { ".m4u":"video\/vnd.mpegurl" }, - { + { ".m4v":"video\/x-m4v" }, - { + { ".ma":"application\/mathematica" }, - { + { ".mag":"application\/vnd.ecowin.chart" }, - { + { ".maker":"application\/vnd.framemaker" }, - { + { ".man":"text\/troff" }, - { + { ".markdown":"text\/x-markdown" }, - { + { ".mathml":"application\/mathml+xml" }, - { + { ".mb":"application\/mathematica" }, - { + { ".mbk":"application\/vnd.mobius.mbk" }, - { + { ".mbox":"application\/mbox" }, - { + { ".mc1":"application\/vnd.medcalcdata" }, - { + { ".mcd":"application\/vnd.mcd" }, - { + { ".mcurl":"text\/vnd.curl.mcurl" }, - { + { ".md":"text\/x-markdown" }, - { + { ".mdb":"application\/x-msaccess" }, - { + { ".mdi":"image\/vnd.ms-modi" }, - { + { ".me":"text\/troff" }, - { + { ".mesh":"model\/mesh" }, - { + { ".mfm":"application\/vnd.mfmp" }, - { + { ".mgz":"application\/vnd.proteus.magazine" }, - { + { ".mht":"message\/rfc822" }, - { + { ".mhtml":"message\/rfc822" }, - { + { ".mid":"audio\/midi" }, - { + { ".midi":"audio\/midi" }, - { + { ".mif":"application\/vnd.mif" }, - { + { ".mime":"message\/rfc822" }, - { + { ".mj2":"video\/mj2" }, - { + { ".mjp2":"video\/mj2" }, - { + { ".mlp":"application\/vnd.dolby.mlp" }, - { + { ".mmd":"application\/vnd.chipnuts.karaoke-mmd" }, - { + { ".mmf":"application\/vnd.smaf" }, - { + { ".mmr":"image\/vnd.fujixerox.edmics-mmr" }, - { + { ".mny":"application\/x-msmoney" }, - { + { ".mobi":"application\/x-mobipocket-ebook" }, - { + { ".mov":"video\/quicktime" }, - { + { ".movie":"video\/x-sgi-movie" }, - { + { ".mp2":"audio\/mpeg" }, - { + { ".mp2a":"audio\/mpeg" }, - { + { ".mp3":"audio\/mpeg" }, - { + { ".mp4":"video\/mp4" }, - { + { ".mp4a":"audio\/mp4" }, - { + { ".mp4s":"application\/mp4" }, - { + { ".mp4v":"video\/mp4" }, - { + { ".mpa":"video\/mpeg" }, - { + { ".mpc":"application\/vnd.mophun.certificate" }, - { + { ".mpe":"video\/mpeg" }, - { + { ".mpeg":"video\/mpeg" }, - { + { ".mpg":"video\/mpeg" }, - { + { ".mpg4":"video\/mp4" }, - { + { ".mpga":"audio\/mpeg" }, - { + { ".mpkg":"application\/vnd.apple.installer+xml" }, - { + { ".mpm":"application\/vnd.blueice.multipass" }, - { + { ".mpn":"application\/vnd.mophun.application" }, - { + { ".mpp":"application\/vnd.ms-project" }, - { + { ".mpt":"application\/vnd.ms-project" }, - { + { ".mpy":"application\/vnd.ibm.minipay" }, - { + { ".mqy":"application\/vnd.mobius.mqy" }, - { + { ".mrc":"application\/marc" }, - { + { ".ms":"text\/troff" }, - { + { ".mscml":"application\/mediaservercontrol+xml" }, - { + { ".mseed":"application\/vnd.fdsn.mseed" }, - { + { ".mseq":"application\/vnd.mseq" }, - { + { ".msf":"application\/vnd.epson.msf" }, - { + { ".msh":"model\/mesh" }, - { + { ".msi":"application\/x-msdownload" }, - { + { ".msl":"application\/vnd.mobius.msl" }, - { + { ".msty":"application\/vnd.muvee.style" }, - { + { ".mts":"model\/vnd.mts" }, - { + { ".mus":"application\/vnd.musician" }, - { + { ".musicxml":"application\/vnd.recordare.musicxml+xml" }, - { + { ".mvb":"application\/x-msmediaview" }, - { + { ".mwf":"application\/vnd.mfer" }, - { + { ".mxf":"application\/mxf" }, - { + { ".mxl":"application\/vnd.recordare.musicxml" }, - { + { ".mxml":"application\/xv+xml" }, - { + { ".mxs":"application\/vnd.triscape.mxs" }, - { + { ".mxu":"video\/vnd.mpegurl" }, - { + { ".n-gage":"application\/vnd.nokia.n-gage.symbian.install" }, - { + { ".nb":"application\/mathematica" }, - { + { ".nc":"application\/x-netcdf" }, - { + { ".ncx":"application\/x-dtbncx+xml" }, - { + { ".ngdat":"application\/vnd.nokia.n-gage.data" }, - { + { ".nlu":"application\/vnd.neurolanguage.nlu" }, - { + { ".nml":"application\/vnd.enliven" }, - { + { ".nnd":"application\/vnd.noblenet-directory" }, - { + { ".nns":"application\/vnd.noblenet-sealer" }, - { + { ".nnw":"application\/vnd.noblenet-web" }, - { + { ".npx":"image\/vnd.net-fpx" }, - { + { ".nsf":"application\/vnd.lotus-notes" }, - { + { ".nws":"message\/rfc822" }, - { + { ".o":"application\/octet-stream" }, - { + { ".oa2":"application\/vnd.fujitsu.oasys2" }, - { + { ".oa3":"application\/vnd.fujitsu.oasys3" }, - { + { ".oas":"application\/vnd.fujitsu.oasys" }, - { + { ".obd":"application\/x-msbinder" }, - { + { ".obj":"application\/octet-stream" }, - { + { ".oda":"application\/oda" }, - { + { ".odb":"application\/vnd.oasis.opendocument.database" }, - { + { ".odc":"application\/vnd.oasis.opendocument.chart" }, - { + { ".odf":"application\/vnd.oasis.opendocument.formula" }, - { + { ".odft":"application\/vnd.oasis.opendocument.formula-template" }, - { + { ".odg":"application\/vnd.oasis.opendocument.graphics" }, - { + { ".odi":"application\/vnd.oasis.opendocument.image" }, - { + { ".odp":"application\/vnd.oasis.opendocument.presentation" }, - { + { ".ods":"application\/vnd.oasis.opendocument.spreadsheet" }, - { + { ".odt":"application\/vnd.oasis.opendocument.text" }, - { + { ".oga":"audio\/ogg" }, - { + { ".ogg":"audio\/ogg" }, - { + { ".ogv":"video\/ogg" }, - { + { ".ogx":"application\/ogg" }, - { + { ".onepkg":"application\/onenote" }, - { + { ".onetmp":"application\/onenote" }, - { + { ".onetoc":"application\/onenote" }, - { + { ".onetoc2":"application\/onenote" }, - { + { ".opf":"application\/oebps-package+xml" }, - { + { ".oprc":"application\/vnd.palm" }, - { + { ".org":"application\/vnd.lotus-organizer" }, - { + { ".osf":"application\/vnd.yamaha.openscoreformat" }, - { + { ".osfpvg":"application\/vnd.yamaha.openscoreformat.osfpvg+xml" }, - { + { ".otc":"application\/vnd.oasis.opendocument.chart-template" }, - { + { ".otf":"application\/x-font-otf" }, - { + { ".otg":"application\/vnd.oasis.opendocument.graphics-template" }, - { + { ".oth":"application\/vnd.oasis.opendocument.text-web" }, - { + { ".oti":"application\/vnd.oasis.opendocument.image-template" }, - { + { ".otm":"application\/vnd.oasis.opendocument.text-master" }, - { + { ".otp":"application\/vnd.oasis.opendocument.presentation-template" }, - { + { ".ots":"application\/vnd.oasis.opendocument.spreadsheet-template" }, - { + { ".ott":"application\/vnd.oasis.opendocument.text-template" }, - { + { ".oxt":"application\/vnd.openofficeorg.extension" }, - { + { ".p":"text\/x-pascal" }, - { + { ".p10":"application\/pkcs10" }, - { + { ".p12":"application\/x-pkcs12" }, - { + { ".p7b":"application\/x-pkcs7-certificates" }, - { + { ".p7c":"application\/pkcs7-mime" }, - { + { ".p7m":"application\/pkcs7-mime" }, - { + { ".p7r":"application\/x-pkcs7-certreqresp" }, - { + { ".p7s":"application\/pkcs7-signature" }, - { + { ".pas":"text\/x-pascal" }, - { + { ".pbd":"application\/vnd.powerbuilder6" }, - { + { ".pbm":"image\/x-portable-bitmap" }, - { + { ".pcf":"application\/x-font-pcf" }, - { + { ".pcl":"application\/vnd.hp-pcl" }, - { + { ".pclxl":"application\/vnd.hp-pclxl" }, - { + { ".pct":"image\/x-pict" }, - { + { ".pcurl":"application\/vnd.curl.pcurl" }, - { + { ".pcx":"image\/x-pcx" }, - { + { ".pdb":"application\/vnd.palm" }, - { + { ".pdf":"application\/pdf" }, - { + { ".pfa":"application\/x-font-type1" }, - { + { ".pfb":"application\/x-font-type1" }, - { + { ".pfm":"application\/x-font-type1" }, - { + { ".pfr":"application\/font-tdpfr" }, - { + { ".pfx":"application\/x-pkcs12" }, - { + { ".pgm":"image\/x-portable-graymap" }, - { + { ".pgn":"application\/x-chess-pgn" }, - { + { ".pgp":"application\/pgp-encrypted" }, - { + { ".pic":"image\/x-pict" }, - { + { ".pkg":"application\/octet-stream" }, - { + { ".pki":"application\/pkixcmp" }, - { + { ".pkipath":"application\/pkix-pkipath" }, - { + { ".pl":"text\/plain" }, - { + { ".plb":"application\/vnd.3gpp.pic-bw-large" }, - { + { ".plc":"application\/vnd.mobius.plc" }, - { + { ".plf":"application\/vnd.pocketlearn" }, - { + { ".pls":"application\/pls+xml" }, - { + { ".pml":"application\/vnd.ctc-posml" }, - { + { ".png":"image\/png" }, - { + { ".pnm":"image\/x-portable-anymap" }, - { + { ".portpkg":"application\/vnd.macports.portpkg" }, - { + { ".pot":"application\/vnd.ms-powerpoint" }, - { + { ".potm":"application\/vnd.ms-powerpoint.template.macroenabled.12" }, - { + { ".potx":"application\/vnd.openxmlformats-officedocument.presentationml.template" }, - { + { ".ppa":"application\/vnd.ms-powerpoint" }, - { + { ".ppam":"application\/vnd.ms-powerpoint.addin.macroenabled.12" }, - { + { ".ppd":"application\/vnd.cups-ppd" }, - { + { ".ppm":"image\/x-portable-pixmap" }, - { + { ".pps":"application\/vnd.ms-powerpoint" }, - { + { ".ppsm":"application\/vnd.ms-powerpoint.slideshow.macroenabled.12" }, - { + { ".ppsx":"application\/vnd.openxmlformats-officedocument.presentationml.slideshow" }, - { + { ".ppt":"application\/vnd.ms-powerpoint" }, - { + { ".pptm":"application\/vnd.ms-powerpoint.presentation.macroenabled.12" }, - { + { ".pptx":"application\/vnd.openxmlformats-officedocument.presentationml.presentation" }, - { + { ".pqa":"application\/vnd.palm" }, - { + { ".prc":"application\/x-mobipocket-ebook" }, - { + { ".pre":"application\/vnd.lotus-freelance" }, - { + { ".prf":"application\/pics-rules" }, - { + { ".ps":"application\/postscript" }, - { + { ".psb":"application\/vnd.3gpp.pic-bw-small" }, - { + { ".psd":"image\/vnd.adobe.photoshop" }, - { + { ".psf":"application\/x-font-linux-psf" }, - { + { ".ptid":"application\/vnd.pvi.ptid1" }, - { + { ".pub":"application\/x-mspublisher" }, - { + { ".pvb":"application\/vnd.3gpp.pic-bw-var" }, - { + { ".pwn":"application\/vnd.3m.post-it-notes" }, - { + { ".pwz":"application\/vnd.ms-powerpoint" }, - { + { ".py":"text\/x-python" }, - { + { ".pya":"audio\/vnd.ms-playready.media.pya" }, - { + { ".pyc":"application\/x-python-code" }, - { + { ".pyo":"application\/x-python-code" }, - { + { ".pyv":"video\/vnd.ms-playready.media.pyv" }, - { + { ".qam":"application\/vnd.epson.quickanime" }, - { + { ".qbo":"application\/vnd.intu.qbo" }, - { + { ".qfx":"application\/vnd.intu.qfx" }, - { + { ".qps":"application\/vnd.publishare-delta-tree" }, - { + { ".qt":"video\/quicktime" }, - { + { ".qwd":"application\/vnd.quark.quarkxpress" }, - { + { ".qwt":"application\/vnd.quark.quarkxpress" }, - { + { ".qxb":"application\/vnd.quark.quarkxpress" }, - { + { ".qxd":"application\/vnd.quark.quarkxpress" }, - { + { ".qxl":"application\/vnd.quark.quarkxpress" }, - { + { ".qxt":"application\/vnd.quark.quarkxpress" }, - { + { ".ra":"audio\/x-pn-realaudio" }, - { + { ".ram":"audio\/x-pn-realaudio" }, - { + { ".rar":"application\/x-rar-compressed" }, - { + { ".ras":"image\/x-cmu-raster" }, - { + { ".rcprofile":"application\/vnd.ipunplugged.rcprofile" }, - { + { ".rdf":"application\/rdf+xml" }, - { + { ".rdz":"application\/vnd.data-vision.rdz" }, - { + { ".rep":"application\/vnd.businessobjects" }, - { + { ".res":"application\/x-dtbresource+xml" }, - { + { ".rgb":"image\/x-rgb" }, - { + { ".rif":"application\/reginfo+xml" }, - { + { ".rl":"application\/resource-lists+xml" }, - { + { ".rlc":"image\/vnd.fujixerox.edmics-rlc" }, - { + { ".rld":"application\/resource-lists-diff+xml" }, - { + { ".rm":"application\/vnd.rn-realmedia" }, - { + { ".rmi":"audio\/midi" }, - { + { ".rmp":"audio\/x-pn-realaudio-plugin" }, - { + { ".rms":"application\/vnd.jcp.javame.midlet-rms" }, - { + { ".rnc":"application\/relax-ng-compact-syntax" }, - { + { ".roff":"text\/troff" }, - { + { ".rpm":"application\/x-rpm" }, - { + { ".rpss":"application\/vnd.nokia.radio-presets" }, - { + { ".rpst":"application\/vnd.nokia.radio-preset" }, - { + { ".rq":"application\/sparql-query" }, - { + { ".rs":"application\/rls-services+xml" }, - { + { ".rsd":"application\/rsd+xml" }, - { + { ".rss":"application\/rss+xml" }, - { + { ".rtf":"application\/rtf" }, - { + { ".rtx":"text\/richtext" }, - { + { ".s":"text\/x-asm" }, - { + { ".saf":"application\/vnd.yamaha.smaf-audio" }, { ".sass":"text/x-sass" }, - { + { ".sbml":"application\/sbml+xml" }, - { + { ".sc":"application\/vnd.ibm.secure-container" }, - { + { ".scd":"application\/x-msschedule" }, - { + { ".scm":"application\/vnd.lotus-screencam" }, - { + { ".scq":"application\/scvp-cv-request" }, - { + { ".scs":"application\/scvp-cv-response" }, { ".scss":"text/x-scss" }, - { + { ".scurl":"text\/vnd.curl.scurl" }, - { + { ".sda":"application\/vnd.stardivision.draw" }, - { + { ".sdc":"application\/vnd.stardivision.calc" }, - { + { ".sdd":"application\/vnd.stardivision.impress" }, - { + { ".sdkd":"application\/vnd.solent.sdkm+xml" }, - { + { ".sdkm":"application\/vnd.solent.sdkm+xml" }, - { + { ".sdp":"application\/sdp" }, - { + { ".sdw":"application\/vnd.stardivision.writer" }, - { + { ".see":"application\/vnd.seemail" }, - { + { ".seed":"application\/vnd.fdsn.seed" }, - { + { ".sema":"application\/vnd.sema" }, - { + { ".semd":"application\/vnd.semd" }, - { + { ".semf":"application\/vnd.semf" }, - { + { ".ser":"application\/java-serialized-object" }, - { + { ".setpay":"application\/set-payment-initiation" }, - { + { ".setreg":"application\/set-registration-initiation" }, - { + { ".sfd-hdstx":"application\/vnd.hydrostatix.sof-data" }, - { + { ".sfs":"application\/vnd.spotfire.sfs" }, - { + { ".sgl":"application\/vnd.stardivision.writer-global" }, - { + { ".sgm":"text\/sgml" }, - { + { ".sgml":"text\/sgml" }, - { + { ".sh":"application\/x-sh" }, - { + { ".shar":"application\/x-shar" }, - { + { ".shf":"application\/shf+xml" }, - { + { ".si":"text\/vnd.wap.si" }, - { + { ".sic":"application\/vnd.wap.sic" }, - { + { ".sig":"application\/pgp-signature" }, - { + { ".silo":"model\/mesh" }, - { + { ".sis":"application\/vnd.symbian.install" }, - { + { ".sisx":"application\/vnd.symbian.install" }, - { + { ".sit":"application\/x-stuffit" }, - { + { ".sitx":"application\/x-stuffitx" }, - { + { ".skd":"application\/vnd.koan" }, - { + { ".skm":"application\/vnd.koan" }, - { + { ".skp":"application\/vnd.koan" }, - { + { ".skt":"application\/vnd.koan" }, - { + { ".sl":"text\/vnd.wap.sl" }, - { + { ".slc":"application\/vnd.wap.slc" }, - { + { ".sldm":"application\/vnd.ms-powerpoint.slide.macroenabled.12" }, - { + { ".sldx":"application\/vnd.openxmlformats-officedocument.presentationml.slide" }, - { + { ".slt":"application\/vnd.epson.salt" }, - { + { ".smf":"application\/vnd.stardivision.math" }, - { + { ".smi":"application\/smil+xml" }, - { + { ".smil":"application\/smil+xml" }, - { + { ".snd":"audio\/basic" }, - { + { ".snf":"application\/x-font-snf" }, - { + { ".so":"application\/octet-stream" }, - { + { ".spc":"application\/x-pkcs7-certificates" }, - { + { ".spf":"application\/vnd.yamaha.smaf-phrase" }, - { + { ".spl":"application\/x-futuresplash" }, - { + { ".spot":"text\/vnd.in3d.spot" }, - { + { ".spp":"application\/scvp-vp-response" }, - { + { ".spq":"application\/scvp-vp-request" }, - { + { ".spx":"audio\/ogg" }, - { + { ".src":"application\/x-wais-source" }, - { + { ".srx":"application\/sparql-results+xml" }, - { + { ".sse":"application\/vnd.kodak-descriptor" }, - { + { ".ssf":"application\/vnd.epson.ssf" }, - { + { ".ssml":"application\/ssml+xml" }, - { + { ".stc":"application\/vnd.sun.xml.calc.template" }, - { + { ".std":"application\/vnd.sun.xml.draw.template" }, - { + { ".stf":"application\/vnd.wt.stf" }, - { + { ".sti":"application\/vnd.sun.xml.impress.template" }, - { + { ".stk":"application\/hyperstudio" }, - { + { ".stl":"application\/vnd.ms-pki.stl" }, - { + { ".str":"application\/vnd.pg.format" }, - { + { ".stw":"application\/vnd.sun.xml.writer.template" }, - { + { ".sus":"application\/vnd.sus-calendar" }, - { + { ".susp":"application\/vnd.sus-calendar" }, - { + { ".sv4cpio":"application\/x-sv4cpio" }, - { + { ".sv4crc":"application\/x-sv4crc" }, - { + { ".svd":"application\/vnd.svd" }, - { + { ".svg":"image\/svg+xml" }, - { + { ".svgz":"image\/svg+xml" }, - { + { ".swa":"application\/x-director" }, - { + { ".swf":"application\/x-shockwave-flash" }, - { + { ".swi":"application\/vnd.arastra.swi" }, - { + { ".sxc":"application\/vnd.sun.xml.calc" }, - { + { ".sxd":"application\/vnd.sun.xml.draw" }, - { + { ".sxg":"application\/vnd.sun.xml.writer.global" }, - { + { ".sxi":"application\/vnd.sun.xml.impress" }, - { + { ".sxm":"application\/vnd.sun.xml.math" }, - { + { ".sxw":"application\/vnd.sun.xml.writer" }, - { + { ".t":"text\/troff" }, - { + { ".tao":"application\/vnd.tao.intent-module-archive" }, - { + { ".tar":"application\/x-tar" }, - { + { ".tcap":"application\/vnd.3gpp2.tcap" }, - { + { ".tcl":"application\/x-tcl" }, - { + { ".teacher":"application\/vnd.smart.teacher" }, - { + { ".tex":"application\/x-tex" }, - { + { ".texi":"application\/x-texinfo" }, - { + { ".texinfo":"application\/x-texinfo" }, - { + { ".text":"text\/plain" }, - { + { ".tfm":"application\/x-tex-tfm" }, - { + { ".tgz":"application\/x-gzip" }, - { + { ".tif":"image\/tiff" }, - { + { ".tiff":"image\/tiff" }, - { + { ".tmo":"application\/vnd.tmobile-livetv" }, - { + { ".torrent":"application\/x-bittorrent" }, - { + { ".tpl":"application\/vnd.groove-tool-template" }, - { + { ".tpt":"application\/vnd.trid.tpt" }, - { + { ".tr":"text\/troff" }, - { + { ".tra":"application\/vnd.trueapp" }, - { + { ".trm":"application\/x-msterminal" }, - { + { ".tsv":"text\/tab-separated-values" }, - { + { ".ttc":"application\/x-font-ttf" }, - { + { ".ttf":"application\/x-font-ttf" }, - { + { ".twd":"application\/vnd.simtech-mindmapper" }, - { + { ".twds":"application\/vnd.simtech-mindmapper" }, - { + { ".txd":"application\/vnd.genomatix.tuxedo" }, - { + { ".txf":"application\/vnd.mobius.txf" }, - { + { ".txt":"text\/plain" }, - { + { ".u32":"application\/x-authorware-bin" }, - { + { ".udeb":"application\/x-debian-package" }, - { + { ".ufd":"application\/vnd.ufdl" }, - { + { ".ufdl":"application\/vnd.ufdl" }, - { + { ".umj":"application\/vnd.umajin" }, - { + { ".unityweb":"application\/vnd.unity" }, - { + { ".uoml":"application\/vnd.uoml+xml" }, - { + { ".uri":"text\/uri-list" }, - { + { ".uris":"text\/uri-list" }, - { + { ".urls":"text\/uri-list" }, - { + { ".ustar":"application\/x-ustar" }, - { + { ".utz":"application\/vnd.uiq.theme" }, - { + { ".uu":"text\/x-uuencode" }, - { + { ".vcd":"application\/x-cdlink" }, - { + { ".vcf":"text\/x-vcard" }, - { + { ".vcg":"application\/vnd.groove-vcard" }, - { + { ".vcs":"text\/x-vcalendar" }, - { + { ".vcx":"application\/vnd.vcx" }, - { + { ".vis":"application\/vnd.visionary" }, - { + { ".viv":"video\/vnd.vivo" }, - { + { ".vor":"application\/vnd.stardivision.writer" }, - { + { ".vox":"application\/x-authorware-bin" }, - { + { ".vrml":"model\/vrml" }, - { + { ".vsd":"application\/vnd.visio" }, - { + { ".vsf":"application\/vnd.vsf" }, - { + { ".vss":"application\/vnd.visio" }, - { + { ".vst":"application\/vnd.visio" }, - { + { ".vsw":"application\/vnd.visio" }, - { + { ".vtu":"model\/vnd.vtu" }, - { + { ".vxml":"application\/voicexml+xml" }, - { + { ".w3d":"application\/x-director" }, - { + { ".wad":"application\/x-doom" }, - { + { ".wav":"audio\/x-wav" }, - { + { ".wax":"audio\/x-ms-wax" }, - { + { ".wbmp":"image\/vnd.wap.wbmp" }, - { + { ".wbs":"application\/vnd.criticaltools.wbs+xml" }, - { + { ".wbxml":"application\/vnd.wap.wbxml" }, - { + { ".wcm":"application\/vnd.ms-works" }, - { + { ".wdb":"application\/vnd.ms-works" }, - { + { ".wiz":"application\/msword" }, - { + { ".wks":"application\/vnd.ms-works" }, - { + { ".wm":"video\/x-ms-wm" }, - { + { ".wma":"audio\/x-ms-wma" }, - { + { ".wmd":"application\/x-ms-wmd" }, - { + { ".wmf":"application\/x-msmetafile" }, - { + { ".wml":"text\/vnd.wap.wml" }, - { + { ".wmlc":"application\/vnd.wap.wmlc" }, - { + { ".wmls":"text\/vnd.wap.wmlscript" }, - { + { ".wmlsc":"application\/vnd.wap.wmlscriptc" }, - { + { ".wmv":"video\/x-ms-wmv" }, - { + { ".wmx":"video\/x-ms-wmx" }, - { + { ".wmz":"application\/x-ms-wmz" }, - { + { ".wpd":"application\/vnd.wordperfect" }, - { + { ".wpl":"application\/vnd.ms-wpl" }, - { + { ".wps":"application\/vnd.ms-works" }, - { + { ".wqd":"application\/vnd.wqd" }, - { + { ".wri":"application\/x-mswrite" }, - { + { ".wrl":"model\/vrml" }, - { + { ".wsdl":"application\/wsdl+xml" }, - { + { ".wspolicy":"application\/wspolicy+xml" }, - { + { ".wtb":"application\/vnd.webturbo" }, - { + { ".wvx":"video\/x-ms-wvx" }, - { + { ".x32":"application\/x-authorware-bin" }, - { + { ".x3d":"application\/vnd.hzn-3d-crossword" }, - { + { ".xap":"application\/x-silverlight-app" }, - { + { ".xar":"application\/vnd.xara" }, - { + { ".xbap":"application\/x-ms-xbap" }, - { + { ".xbd":"application\/vnd.fujixerox.docuworks.binder" }, - { + { ".xbm":"image\/x-xbitmap" }, - { + { ".xdm":"application\/vnd.syncml.dm+xml" }, - { + { ".xdp":"application\/vnd.adobe.xdp+xml" }, - { + { ".xdw":"application\/vnd.fujixerox.docuworks" }, - { + { ".xenc":"application\/xenc+xml" }, - { + { ".xer":"application\/patch-ops-error+xml" }, - { + { ".xfdf":"application\/vnd.adobe.xfdf" }, - { + { ".xfdl":"application\/vnd.xfdl" }, - { + { ".xht":"application\/xhtml+xml" }, - { + { ".xhtml":"application\/xhtml+xml" }, - { + { ".xhvml":"application\/xv+xml" }, - { + { ".xif":"image\/vnd.xiff" }, - { + { ".xla":"application\/vnd.ms-excel" }, - { + { ".xlam":"application\/vnd.ms-excel.addin.macroenabled.12" }, - { + { ".xlb":"application\/vnd.ms-excel" }, - { + { ".xlc":"application\/vnd.ms-excel" }, - { + { ".xlm":"application\/vnd.ms-excel" }, - { + { ".xls":"application\/vnd.ms-excel" }, - { + { ".xlsb":"application\/vnd.ms-excel.sheet.binary.macroenabled.12" }, - { + { ".xlsm":"application\/vnd.ms-excel.sheet.macroenabled.12" }, - { + { ".xlsx":"application\/vnd.openxmlformats-officedocument.spreadsheetml.sheet" }, - { + { ".xlt":"application\/vnd.ms-excel" }, - { + { ".xltm":"application\/vnd.ms-excel.template.macroenabled.12" }, - { + { ".xltx":"application\/vnd.openxmlformats-officedocument.spreadsheetml.template" }, - { + { ".xlw":"application\/vnd.ms-excel" }, - { + { ".xml":"application\/xml" }, - { + { ".xo":"application\/vnd.olpc-sugar" }, - { + { ".xop":"application\/xop+xml" }, - { + { ".xpdl":"application\/xml" }, - { + { ".xpi":"application\/x-xpinstall" }, - { + { ".xpm":"image\/x-xpixmap" }, - { + { ".xpr":"application\/vnd.is-xpr" }, - { + { ".xps":"application\/vnd.ms-xpsdocument" }, - { + { ".xpw":"application\/vnd.intercon.formnet" }, - { + { ".xpx":"application\/vnd.intercon.formnet" }, - { + { ".xsl":"application\/xml" }, - { + { ".xslt":"application\/xslt+xml" }, - { + { ".xsm":"application\/vnd.syncml+xml" }, - { + { ".xspf":"application\/xspf+xml" }, - { + { ".xul":"application\/vnd.mozilla.xul+xml" }, - { + { ".xvm":"application\/xv+xml" }, - { + { ".xvml":"application\/xv+xml" }, - { + { ".xwd":"image\/x-xwindowdump" }, - { + { ".xyz":"chemical\/x-xyz" }, - { + { ".zaz":"application\/vnd.zzazz.deck+xml" }, - { + { ".zip":"application\/zip" }, - { + { ".zir":"application\/vnd.zul" }, - { + { ".zirz":"application\/vnd.zul" }, - { + { ".zmm":"application\/vnd.handheld-entertainment+xml" } ]
@@ -9,7 +9,7 @@ for item in json.items:
for pair in item.pairs: result[$pair.key] = $pair.val.getStr -var CONTENT_TYPES* {.threadvar.}: StringTableRef +var CONTENT_TYPES* {.threadvar.}: StringTableRef CONTENT_TYPES = loadContentTypes()
@@ -1,11 +1,11 @@
-import - x_sqlite3, +import + x_sqlite3, x_db_sqlite as db, - strutils, + strutils, os, oids, json, - pegs, + pegs, strtabs, strutils, base64,@@ -27,13 +27,13 @@ db.exec SQL_CREATE_INDEX_DOCUMENTS_ID
db.exec SQL_CREATE_INDEX_TAGS_TAG_ID db.exec SQL_CREATE_INDEX_TAGS_DOCUMENT_ID -proc dropIndexes(db: DbConn) = +proc dropIndexes(db: DbConn) = db.exec SQL_DROP_INDEX_DOCUMENTS_DOCID db.exec SQL_DROP_INDEX_DOCUMENTS_ID db.exec SQL_DROP_INDEX_TAGS_TAG_ID db.exec SQL_DROP_INDEX_TAGS_DOCUMENT_ID -proc createDatastore*(file:string) = +proc createDatastore*(file:string) = if file.fileExists(): raise newException(EDatastoreExists, "Datastore '$1' already exists." % file) LOG.debug("Creating datastore '$1'", file)@@ -48,7 +48,7 @@ LOG.debug("Creating indexes")
data.createIndexes() LOG.debug("Database created") -proc closeDatastore*(store: Datastore) = +proc closeDatastore*(store: Datastore) = try: db.close(store.db) except:@@ -69,7 +69,7 @@ try:
result.db = db.open(file, "", "", "") # Register custom function & PRAGMAs LOG.debug("Registering custom functions...") - discard create_function(cast[PSqlite3](result.db), "rank".cstring, -1, SQLITE_ANY, cast[pointer](SQLITE_DETERMINISTIC), okapi_bm25f_kb, nil, nil) + discard create_function(cast[PSqlite3](result.db), "rank", -1, SQLITE_ANY, cast[pointer](SQLITE_DETERMINISTIC), okapi_bm25f_kb, nil, nil) LOG.debug("Executing PRAGMAs...") discard result.db.tryExec("PRAGMA locking_mode = exclusive".sql) discard result.db.tryExec("PRAGMA page_size = 4096".sql)@@ -99,12 +99,14 @@ if LS_TRANSACTION:
LOG.debug("Committing transaction") LS_TRANSACTION = false store.db.exec("COMMIT".sql) + LOG.debug("Committed.") proc rollback(store: Datastore) = if LS_TRANSACTION: LOG.debug("Rolling back transaction") LS_TRANSACTION = false store.db.exec("ROLLBACK".sql) + LOG.debug("Rolled back.") # Manage Tags@@ -194,7 +196,7 @@ var data = rawdata
if contenttype == "application/json": # Validate JSON data try: - discard data.parseJson + discard data.parseJson except: raise newException(JsonParsingError, "Invalid JSON content - " & getCurrentExceptionMsg()) if id == "":@@ -226,7 +228,7 @@ store.commit()
return $store.retrieveRawDocument(id) except: store.rollback() - eWarn() + eWarn() raise proc updateDocument*(store: Datastore, id: string, rawdata: string, contenttype = "text/plain", binary = -1, searchable = 1): string =@@ -237,7 +239,7 @@ var data = rawdata
if contenttype == "application/json": # Validate JSON data try: - discard data.parseJson + discard data.parseJson except: raise newException(JsonParsingError, "Invalid JSON content - " & getCurrentExceptionMsg()) var searchable = searchable@@ -382,7 +384,7 @@ raise newException(EDirectoryNotFound, "Directory '$1' not found." % dir)
for f in dir.walkDirRec(): if f.existsDir: continue - if f.splitFile.name.startsWith("."): + if f.splitFile.name.startsWith("."): # Ignore hidden files continue files.add(f)@@ -395,7 +397,7 @@ store.begin()
LOG.info("Importing $1 files in $2 batches", files.len, nBatches) LOG.debug("Dropping column indexes...") store.db.dropIndexes() - for f in files: + for f in files: try: store.importFile(f, dir) cFiles.inc
@@ -1,4 +1,4 @@
-import +import strutils, times@@ -18,19 +18,19 @@ stderr.writeLine(currentTime(true) & " " & kind & ": " & s)
else: echo currentTime(true), " ", kind, ": ", s -proc error*(logger: Logger, message: string, params: varargs[string, `$`]) = +proc error*(logger: Logger, message: string, params: varargs[string, `$`]) = if logger.level <= lvError: logger.msg(" ERROR", message, params) -proc warn*(logger: Logger, message: string, params: varargs[string, `$`]) = +proc warn*(logger: Logger, message: string, params: varargs[string, `$`]) = if logger.level <= lvWarn: logger.msg("WARNING", message, params) -proc info*(logger: Logger, message: string, params: varargs[string, `$`]) = +proc info*(logger: Logger, message: string, params: varargs[string, `$`]) = if logger.level <= lvInfo: logger.msg(" INFO", message, params) -proc debug*(logger: Logger, message: string, params: varargs[string, `$`]) = +proc debug*(logger: Logger, message: string, params: varargs[string, `$`]) = if logger.level <= lvDebug: logger.msg(" DEBUG", message, params)
@@ -20,21 +20,21 @@ SQL_CREATE_INDEX_DOCUMENTS_ID* = sql"CREATE INDEX IF NOT EXISTS documents_id ON documents(id)"
SQL_CREATE_INDEX_TAGS_DOCUMENT_ID* = sql"CREATE INDEX IF NOT EXISTS tags_document_id ON tags(document_id)" SQL_CREATE_INDEX_TAGS_TAG_ID* = sql"CREATE INDEX IF NOT EXISTS tags_tag_id ON tags(tag_id)" - SQL_DROP_INDEX_DOCUMENTS_DOCID* = sql"DROP INDEX IF EXISTS documents_docid" + SQL_DROP_INDEX_DOCUMENTS_DOCID* = sql"DROP INDEX IF EXISTS documents_docid" SQL_DROP_INDEX_DOCUMENTS_ID* = sql"DROP INDEX IF EXISTS documents_id" SQL_DROP_INDEX_TAGS_DOCUMENT_ID* = sql"DROP INDEX IF EXISTS tags_document_id" SQL_DROP_INDEX_TAGS_TAG_ID* = sql"DROP INDEX IF EXISTS tags_tag_id" - + SQL_REINDEX* = sql"REINDEX" SQL_OPTIMIZE* = sql"INSERT INTO searchdata(searchdata) VALUES('optimize')" SQL_REBUILD* = sql"INSERT INTO searchdata(searchdata) VALUES('rebuild')" - + SQL_VACUUM* = sql"VACUUM" const SQL_CREATE_SEARCHDATA_TABLE* = sql""" CREATE VIRTUAL TABLE searchdata USING fts4( id TEXT UNIQUE NOT NULL, -data TEXT, +data TEXT, tokenize=porter) """@@ -102,7 +102,7 @@ """
const SQL_DELETE_DOCUMENT* = sql""" DELETE FROM documents -WHERE id = ? +WHERE id = ? """ const SQL_INSERT_TAG* = sql"""@@ -161,32 +161,32 @@ tag_id = ?
""" const SQL_SELECT_TAGS_WITH_TOTALS* = sql""" -SELECT DISTINCT tag_id, COUNT(document_id) +SELECT DISTINCT tag_id, COUNT(document_id) FROM tags GROUP BY tag_id ORDER BY tag_id ASC """ const SQL_COUNT_TAGS* = sql""" -SELECT COUNT(DISTINCT tag_id) FROM tags +SELECT COUNT(DISTINCT tag_id) FROM tags """ const SQL_COUNT_DOCUMENTS* = sql""" -SELECT COUNT(docid) FROM documents +SELECT COUNT(docid) FROM documents """ const SQL_DELETE_DOCUMENTS_BY_TAG* = sql""" DELETE FROM documents -WHERE documents.id IN +WHERE documents.id IN (SELECT document_id FROM tags WHERE tag_id = ?) """ const SQL_DELETE_SEARCHDATA_BY_TAG* = sql""" DELETE FROM searchdata -WHERE id IN +WHERE id IN (SELECT document_id FROM tags WHERE tag_id = ?) """ const SQL_DELETE_TAGS_BY_TAG* = sql""" DELETE FROM tags -WHERE document_id IN +WHERE document_id IN (SELECT document_id FROM tags WHERE tag_id = ?) """
@@ -1,10 +1,9 @@
-import +import asynchttpserver, - asyncdispatch, - times, - strutils, - pegs, - strtabs, + asyncdispatch, + times, + strutils, + pegs, logger, cgi, os,@@ -19,7 +18,7 @@ api_v2,
api_v3, api_v4 -export +export api_v4 proc getReqInfo(req: LSRequest): string =@@ -127,7 +126,7 @@ else:
return resError(Http400, "Bad Request - Not serving any directory." % info.version) else: return resError(Http404, "Resource Not Found: $1" % info.resource) - elif info.version == "v1": + elif info.version == "v1": if info.resource.match(peg"^docs / info$"): return api_v1.route(req, LS, info.resource, info.id) elif info.resource.match(peg"^dir$"):@@ -151,7 +150,7 @@ var matches = @["", "", ""]
template route(req: LSRequest, peg: Peg, op: untyped): untyped = if req.url.path.find(peg, matches) != -1: op - try: + try: var info: ResourceInfo req.route peg"^\/?$": info.version = "v4"@@ -195,4 +194,3 @@ echo(LS.appname & " v" & LS.appversion & " started on " & LS.address & ":" & $LS.port & ".")
if LS.mount: echo("Mirroring datastore changes to: " & LS.directory) asyncCheck server.serve(LS.port.Port, handleHttpRequest, LS.address) -
@@ -2,12 +2,11 @@ import
x_db_sqlite, asynchttpserver, pegs, - json, - strtabs + json import config -type +type EDatastoreExists* = object of Exception EDatastoreDoesNotExist* = object of Exception EDatastoreUnavailable* = object of Exception@@ -16,7 +15,6 @@ EDirectoryNotFound* = object of Exception
EFileNotFound* = object of Exception EFileExists* = object of Exception EInvalidRequest* = object of Exception - uarray* {.unchecked.} [T] = array[0..0, T] ExecutionData* = object operation*: string file*: string@@ -32,10 +30,10 @@ tables*: seq[string]
jsonFilter*: string jsonSelect*: seq[tuple[path: string, alias: string]] select*: seq[string] - single*:bool - limit*: int - offset*: int - orderby*: string + single*:bool + limit*: int + offset*: int + orderby*: string tags*: string like*: string createdAfter*: string@@ -49,10 +47,10 @@ tag*: string
startswith*: bool endswith*: bool negated*: bool - Operation* = enum - opRun, - opImport, - opExport, + Operation* = enum + opRun, + opImport, + opExport, opDelete, opVacuum, opOptimize,@@ -91,7 +89,7 @@ id: string,
version: string ] -var +var PEG_TAG* {.threadvar.}: Peg PEG_USER_TAG* {.threadvar.}: Peg PEG_DEFAULT_URL* {.threadvar.}: Peg@@ -111,11 +109,11 @@ LS.appname = appname
TAB_HEADERS = { "Access-Control-Allow-Origin": "*", - "Access-Control-Allow-Headers": "Content-Type, Authorization", + "Access-Control-Allow-Headers": "Authorization, Content-Type", "Server": LS.appname & "/" & LS.appversion } proc newQueryOptions*(): QueryOptions = - return QueryOptions(select: @["documents.id AS id", "documents.data AS data", "content_type", "binary", "searchable", "created", "modified"], - single: false, limit: 0, offset: 0, orderby: "", tags: "", search: "", folder: "", like: "", + return QueryOptions(select: @["documents.id AS id", "documents.data AS data", "content_type", "binary", "searchable", "created", "modified"], + single: false, limit: 0, offset: 0, orderby: "", tags: "", search: "", folder: "", like: "", createdAfter: "", createdBefore: "", modifiedAfter: "", modifiedBefore: "", jsonFilter: "", jsonSelect: newSeq[tuple[path: string, alias: string]](), tables: newSeq[string]())
@@ -1,22 +1,20 @@
-import +import x_sqlite3, - x_db_sqlite, - asynchttpserver, + x_db_sqlite, json, - strutils, - pegs, - asyncdispatch, - math, - sequtils, - strtabs + strutils, + pegs, + asynchttpserver, + math, + sequtils -import - types, - queries, - contenttypes, +import + types, + queries, + contenttypes, logger -proc setOrigin*(LS: LiteStore, req: LSRequest, headers: var HttpHeaders) = +proc setOrigin*(LS: LiteStore, req: LSRequest, headers: var HttpHeaders) = var host = "" var port = "" var protocol = "http"@@ -27,11 +25,20 @@ return
elif req.url.hostname != "" and req.url.port != "": host = req.url.hostname port = req.url.port + elif req.headers.hasKey("origin"): + let parts = req.headers["origin"].split("://") + protocol = parts[0] + let server = parts[1].split(":") + if (server.len >= 2): + host = server[0] + port = server[1] + else: + host = server[0] + port = "80" else: headers["Access-Control-Allow-Origin"] = "*" return - if req.url.scheme != "": - protocol = req.url.scheme + headers["Vary"] = "Origin" headers["Access-Control-Allow-Origin"] = "$1://$2:$3" % [protocol, host, port] proc isFolder*(id: string): bool =@@ -51,7 +58,7 @@ for tag in tags.split(','):
if not tag.match(PEG_TAG): raise newException(EInvalidTag, "Invalid Tag '$1'" % tag) result = result & "AND " & doc_id_col & " IN (" & select_tagged & tag & "') " - + proc prepareSelectDocumentsQuery*(options: var QueryOptions): string = var tables = options.tables result = "SELECT "@@ -63,11 +70,11 @@ options.tags = options.tags.split(",").concat(@["$subtype:json"]).join(",")
if options.search.len > 0: if options.select[0] != "COUNT(docid)": let rank = "rank(matchinfo(searchdata, 'pcxnal'), 1.20, 0.75, 5.0, 0.5) AS rank" - let snippet = "snippet(searchdata, \"<strong>\", \"</strong>\", \"<strong>…</strong>\", -1, 30) as highlight" + let snippet = "snippet(searchdata, \"<strong>\", \"</strong>\", \"<strong>…</strong>\", -1, 30) as highlight" options.select.add(snippet) options.select.add("ranktable.rank AS rank") options.orderby = "rank DESC" - # Create inner select + # Create inner select var innerSelect = "SELECT docid, " & rank & " FROM searchdata WHERE searchdata MATCH '" & options.search.replace("'", "''") & "' " if options.tags.len > 0: innerSelect = innerSelect & options.tags.selectDocumentsByTags()@@ -82,9 +89,14 @@ result = result & " FROM " & tables.join(", ") & " JOIN (" & innerSelect & ") as ranktable USING(docid) JOIN searchdata USING(docid) "
result = result & "WHERE 1=1 " else: tables = options.tables & @["searchdata"] + if options.jsonFilter != "": + options.select[0] = "COUNT(documents.docid)" + tables = tables & @["documents"] result = result & options.select.join(", ") result = result & " FROM "&tables.join(", ")&" " result = result & "WHERE 1=1 " + if options.jsonFilter != "": + result = result & "AND documents.id = searchdata.id " options.orderby = "" else: tables = options.tables & @["documents"]@@ -115,14 +127,14 @@ result = result & "AND " & options.jsonFilter
if options.search.len > 0: result = result & "AND searchdata MATCH '" & options.search.replace("'", "''") & "' " if options.orderby.len > 0 and options.select[0] != "COUNT(docid)": - result = result & "ORDER BY " & options.orderby & " " - if options.limit > 0 and options.search.len == 0: + result = result & "ORDER BY " & options.orderby & " " + if options.limit > 0 and options.search.len == 0: # If searching, do not add limit to the outer select, it's already in the nested select (ranktable) result = result & "LIMIT " & $options.limit & " " if options.offset > 0: result = result & "OFFSET " & $options.offset & " " LOG.debug(result.replace("$", "$$")) - + proc prepareSelectTagsQuery*(options: QueryOptions): string = var group = true if options.select.len > 0 and options.select[0] == "COUNT(tag_id)":@@ -136,9 +148,9 @@ if options.single:
result = result & "WHERE tag_id = ?" elif options.like.len > 0: if options.like[options.like.len-1] == '*': - result = result & "WHERE tag_id BETWEEN ? AND ? " + result = result & "WHERE tag_id BETWEEN ? AND ? " else: - result = result & "WHERE tag_id LIKE ? " + result = result & "WHERE tag_id LIKE ? " if group: result = result & "GROUP BY tag_id " if options.limit > 0:@@ -236,7 +248,7 @@ tags.add "$format:text"
for tag in tags: store.db.exec(SQL_INSERT_TAG, tag, docid) -proc destroyDocumentSystemTags*(store: Datastore, docid: string) = +proc destroyDocumentSystemTags*(store: Datastore, docid: string) = discard store.db.execAffectedRows(SQL_DELETE_DOCUMENT_SYSTEM_TAGS, docid) proc fail*(code: int, msg: string) =@@ -248,7 +260,7 @@ var h = newHttpHeaders(TAB_HEADERS)
h["Content-Type"] = ct return h -proc ctJsonHeader*(): HttpHeaders = +proc ctJsonHeader*(): HttpHeaders = return ctHeader("application/json") proc resError*(code: HttpCode, message: string, trace = ""): LSResponse =@@ -270,7 +282,7 @@ var e = getCurrentException()
LOG.warn(e.msg) LOG.debug(getStackTrace(e)) -proc validate*(req: Request, LS: LiteStore, resource: string, id: string, cb: proc(req: Request, LS: LiteStore, resource: string, id: string):LSResponse): LSResponse = +proc validate*(req: Request, LS: LiteStore, resource: string, id: string, cb: proc(req: Request, LS: LiteStore, resource: string, id: string):LSResponse): LSResponse = if req.reqMethod == HttpPost or req.reqMethod == HttpPut or req.reqMethod == HttpPatch: var ct = "" let body = req.body.strip@@ -297,13 +309,13 @@ # found at: https://github.com/rads/sqlite-okapi-bm25
# which is covered by the MIT License # http://opensource.org/licenses/MIT # the following code shall also be covered by the same MIT License -proc okapi_bm25f_kb*(pCtx: Pcontext, nVal: int32, apVal: PValueArg) {.cdecl.} = - var matchinfo = cast[ptr uarray[int32]](value_blob(apVal[0])) +let okapi_bm25f_kb*: Create_function_func_func = proc (pCtx: Pcontext, nVal: int32, apVal: PValueArg) {.cdecl.} = + var matchinfo = cast[ptr UncheckedArray[int32]](value_blob(apVal[0])) # Setting the default values and ignoring argument based inputs so the extra # arguments can be the column weights instead. if nVal < 2: pCtx.result_error("wrong number of arguments to function okapi_bm25_kb(), expected k1 parameter", -1) - if nVal < 3: + if nVal < 3: pCtx.result_error("wrong number of arguments to function okapi_bm25_kb(), expected b parameter", -1); let K1 = value_double(apVal[1]) # 1.2 let B = value_double(apVal[2]) # 0.75@@ -320,8 +332,8 @@ let N_OFFSET = X_OFFSET + 3*termCount*colCount
let A_OFFSET = N_OFFSET + 1 let L_OFFSET = A_OFFSET + colCount let totalDocs = matchinfo[N_OFFSET].float - var avgLength:float = 0.0 - var docLength:float = 0.0 + var avgLength:float = 0.0 + var docLength:float = 0.0 for col in 0..colCount-1: avgLength = avgLength + matchinfo[A_OFFSET + col].float docLength = docLength + matchinfo[L_OFFSET + col].float
@@ -10,8 +10,14 @@
## A higher level `SQLite`:idx: database wrapper. This interface ## is implemented for other databases too. ## -## See also: `db_odbc <db_odbc.html>`_, `db_postgres <db_postgres.html>`_, -## `db_mysql <db_mysql.html>`_. +## Basic usage +## =========== +## +## The basic flow of using this module is: +## +## 1. Open database connection +## 2. Execute SQL query +## 3. Close database connection ## ## Parameter substitution ## ----------------------@@ -21,91 +27,117 @@ ## That is, using the ``?`` (question mark) to signify the place where a
## value should be placed. For example: ## ## .. code-block:: Nim -## sql"INSERT INTO myTable (colA, colB, colC) VALUES (?, ?, ?)" ## -## Examples -## -------- +## sql"INSERT INTO my_table (colA, colB, colC) VALUES (?, ?, ?)" ## ## Opening a connection to a database -## ================================== +## ---------------------------------- ## ## .. code-block:: Nim -## import db_sqlite -## let db = open("localhost", "user", "password", "dbname") -## db.close() +## +## import db_sqlite +## +## # user, password, database name can be empty. +## # These params are not used on db_sqlite module. +## let db = open("mytest.db", "", "", "") +## db.close() ## ## Creating a table -## ================ +## ---------------- ## ## .. code-block:: Nim -## db.exec(sql"DROP TABLE IF EXISTS myTable") -## db.exec(sql("""CREATE TABLE myTable ( -## id integer, -## name varchar(50) not null)""")) +## +## db.exec(sql"DROP TABLE IF EXISTS my_table") +## db.exec(sql"""CREATE TABLE my_table ( +## id INTEGER, +## name VARCHAR(50) NOT NULL +## )""") ## ## Inserting data -## ============== +## -------------- ## ## .. code-block:: Nim -## db.exec(sql"INSERT INTO myTable (id, name) VALUES (0, ?)", -## "Jack") +## +## db.exec(sql"INSERT INTO my_table (id, name) VALUES (0, ?)", +## "Jack") ## ## Larger example -## ============== +## -------------- ## ## .. code-block:: nim ## -## import db_sqlite, math +## import db_sqlite, math ## -## let theDb = open("mytest.db", nil, nil, nil) +## let db = open("mytest.db", "", "", "") ## -## theDb.exec(sql"Drop table if exists myTestTbl") -## theDb.exec(sql("""create table myTestTbl ( -## Id INTEGER PRIMARY KEY, -## Name VARCHAR(50) NOT NULL, -## i INT(11), -## f DECIMAL(18,10))""")) +## db.exec(sql"DROP TABLE IF EXISTS my_table") +## db.exec(sql"""CREATE TABLE my_table ( +## id INTEGER PRIMARY KEY, +## name VARCHAR(50) NOT NULL, +## i INT(11), +## f DECIMAL(18, 10) +## )""") ## -## theDb.exec(sql"BEGIN") -## for i in 1..1000: -## theDb.exec(sql"INSERT INTO myTestTbl (name,i,f) VALUES (?,?,?)", -## "Item#" & $i, i, sqrt(i.float)) -## theDb.exec(sql"COMMIT") +## db.exec(sql"BEGIN") +## for i in 1..1000: +## db.exec(sql"INSERT INTO my_table (name, i, f) VALUES (?, ?, ?)", +## "Item#" & $i, i, sqrt(i.float)) +## db.exec(sql"COMMIT") +## +## for x in db.fastRows(sql"SELECT * FROM my_table"): +## echo x +## +## let id = db.tryInsertId(sql"""INSERT INTO my_table (name, i, f) +## VALUES (?, ?, ?)""", +## "Item#1001", 1001, sqrt(1001.0)) +## echo "Inserted item: ", db.getValue(sql"SELECT name FROM my_table WHERE id=?", id) ## -## for x in theDb.fastRows(sql"select * from myTestTbl"): -## echo x +## db.close() ## -## let id = theDb.tryInsertId(sql"INSERT INTO myTestTbl (name,i,f) VALUES (?,?,?)", -## "Item#1001", 1001, sqrt(1001.0)) -## echo "Inserted item: ", theDb.getValue(sql"SELECT name FROM myTestTbl WHERE id=?", id) +## See also +## ======== ## -## theDb.close() +## * `db_odbc module <db_odbc.html>`_ for ODBC database wrapper +## * `db_mysql module <db_mysql.html>`_ for MySQL database wrapper +## * `db_postgres module <db_postgres.html>`_ for PostgreSQL database wrapper -{.deadCodeElim:on.} +{.deadCodeElim: on.} # dce option deprecated -import strutils, x_sqlite3 as sqlite3 +import x_sqlite3 as sqlite3 # h3rald import db_common export db_common type - DbConn* = PSqlite3 ## encapsulates a database connection - Row* = seq[string] ## a row of a dataset. NULL database values will be - ## converted to nil. - InstantRow* = Pstmt ## a handle that can be used to get a row's column - ## text on demand -{.deprecated: [TRow: Row, TDbConn: DbConn].} + DbConn* = PSqlite3 ## Encapsulates a database connection. + Row* = seq[string] ## A row of a dataset. `NULL` database values will be + ## converted to an empty string. + InstantRow* = PStmt ## A handle that can be used to get a row's column + ## text on demand. proc dbError*(db: DbConn) {.noreturn.} = - ## raises a DbError exception. + ## Raises a `DbError` exception. + ## + ## **Examples:** + ## + ## .. code-block:: Nim + ## + ## let db = open("mytest.db", "", "", "") + ## if not db.tryExec(sql"SELECT * FROM not_exist_table"): + ## dbError(db) + ## db.close() var e: ref DbError new(e) e.msg = $sqlite3.errmsg(db) raise e proc dbQuote*(s: string): string = - ## DB quotes the string. - if s.len == 0: return "NULL" + ## Escapes the `'` (single quote) char to `''`. + ## Because single quote is used for defining `VARCHAR` in SQL. + runnableExamples: + doAssert dbQuote("'") == "''''" + doAssert dbQuote("A Foobar's pen.") == "'A Foobar''s pen.'" + result = "'" for c in items(s): if c == '\'': add(result, "''")@@ -125,16 +157,40 @@
proc tryExec*(db: DbConn, query: SqlQuery, args: varargs[string, `$`]): bool {. tags: [ReadDbEffect, WriteDbEffect].} = - ## tries to execute the query and returns true if successful, false otherwise. + ## Tries to execute the query and returns `true` if successful, `false` otherwise. + ## + ## **Examples:** + ## + ## .. code-block:: Nim + ## + ## let db = open("mytest.db", "", "", "") + ## if not db.tryExec(sql"SELECT * FROM my_table"): + ## dbError(db) + ## db.close() + assert(not db.isNil, "Database not connected.") var q = dbFormat(query, args) - var stmt: sqlite3.Pstmt + var stmt: sqlite3.PStmt if prepare_v2(db, q, q.len.cint, stmt, nil) == SQLITE_OK: - if step(stmt) == SQLITE_DONE: + let x = step(stmt) + if x in {SQLITE_DONE, SQLITE_ROW}: result = finalize(stmt) == SQLITE_OK proc exec*(db: DbConn, query: SqlQuery, args: varargs[string, `$`]) {. tags: [ReadDbEffect, WriteDbEffect].} = - ## executes the query and raises DbError if not successful. + ## Executes the query and raises a `DbError` exception if not successful. + ## + ## **Examples:** + ## + ## .. code-block:: Nim + ## + ## let db = open("mytest.db", "", "", "") + ## try: + ## db.exec(sql"INSERT INTO my_table (id, name) VALUES (?, ?)", + ## 1, "item#1") + ## except: + ## stderr.writeLine(getCurrentExceptionMsg()) + ## finally: + ## db.close() if not tryExec(db, query, args): dbError(db) proc newRow(L: int): Row =@@ -142,12 +198,13 @@ newSeq(result, L)
for i in 0..L-1: result[i] = "" proc setupQuery(db: DbConn, query: SqlQuery, - args: varargs[string]): Pstmt = + args: varargs[string]): PStmt = + assert(not db.isNil, "Database not connected.") var q = dbFormat(query, args) if prepare_v2(db, q, q.len.cint, result, nil) != SQLITE_OK: dbError(db) -proc setRow(stmt: Pstmt, r: var Row, cols: cint) = - for col in 0..cols-1: +proc setRow(stmt: PStmt, r: var Row, cols: cint) = + for col in 0'i32..cols-1: setLen(r[col], column_bytes(stmt, col)) # set capacity setLen(r[col], 0) let x = column_text(stmt, col)@@ -157,28 +214,82 @@ iterator fastRows*(db: DbConn, query: SqlQuery,
args: varargs[string, `$`]): Row {.tags: [ReadDbEffect].} = ## Executes the query and iterates over the result dataset. ## - ## This is very fast, but potentially dangerous. Use this iterator only + ## This is very fast, but potentially dangerous. Use this iterator only ## if you require **ALL** the rows. ## - ## Breaking the fastRows() iterator during a loop will cause the next - ## database query to raise a DbError exception ``unable to close due to ...``. + ## **Note:** Breaking the `fastRows()` iterator during a loop will cause the + ## next database query to raise a `DbError` exception ``unable to close due + ## to ...``. + ## + ## **Examples:** + ## + ## .. code-block:: Nim + ## + ## let db = open("mytest.db", "", "", "") + ## + ## # Records of my_table: + ## # | id | name | + ## # |----|----------| + ## # | 1 | item#1 | + ## # | 2 | item#2 | + ## + ## for row in db.fastRows(sql"SELECT id, name FROM my_table"): + ## echo row + ## + ## # Output: + ## # @["1", "item#1"] + ## # @["2", "item#2"] + ## + ## db.close() var stmt = setupQuery(db, query, args) var L = (column_count(stmt)) var result = newRow(L) - while step(stmt) == SQLITE_ROW: - setRow(stmt, result, L) - yield result - if finalize(stmt) != SQLITE_OK: dbError(db) + try: + while step(stmt) == SQLITE_ROW: + setRow(stmt, result, L) + yield result + finally: + if finalize(stmt) != SQLITE_OK: dbError(db) iterator instantRows*(db: DbConn, query: SqlQuery, args: varargs[string, `$`]): InstantRow {.tags: [ReadDbEffect].} = - ## same as fastRows but returns a handle that can be used to get column text - ## on demand using []. Returned handle is valid only within the iterator body. + ## Similar to `fastRows iterator <#fastRows.i,DbConn,SqlQuery,varargs[string,]>`_ + ## but returns a handle that can be used to get column text + ## on demand using `[]`. Returned handle is valid only within the iterator body. + ## + ## **Examples:** + ## + ## .. code-block:: Nim + ## + ## let db = open("mytest.db", "", "", "") + ## + ## # Records of my_table: + ## # | id | name | + ## # |----|----------| + ## # | 1 | item#1 | + ## # | 2 | item#2 | + ## + ## for row in db.instantRows(sql"SELECT * FROM my_table"): + ## echo "id:" & row[0] + ## echo "name:" & row[1] + ## echo "length:" & $len(row) + ## + ## # Output: + ## # id:1 + ## # name:item#1 + ## # length:2 + ## # id:2 + ## # name:item#2 + ## # length:2 + ## + ## db.close() var stmt = setupQuery(db, query, args) - while step(stmt) == SQLITE_ROW: - yield stmt - if finalize(stmt) != SQLITE_OK: dbError(db) + try: + while step(stmt) == SQLITE_ROW: + yield stmt + finally: + if finalize(stmt) != SQLITE_OK: dbError(db) proc toTypeKind(t: var DbType; x: int32) = case x@@ -205,26 +316,94 @@
iterator instantRows*(db: DbConn; columns: var DbColumns; query: SqlQuery, args: varargs[string, `$`]): InstantRow {.tags: [ReadDbEffect].} = - ## same as fastRows but returns a handle that can be used to get column text - ## on demand using []. Returned handle is valid only within the iterator body. + ## Similar to `instantRows iterator <#instantRows.i,DbConn,SqlQuery,varargs[string,]>`_, + ## but sets information about columns to `columns`. + ## + ## **Examples:** + ## + ## .. code-block:: Nim + ## + ## let db = open("mytest.db", "", "", "") + ## + ## # Records of my_table: + ## # | id | name | + ## # |----|----------| + ## # | 1 | item#1 | + ## # | 2 | item#2 | + ## + ## var columns: DbColumns + ## for row in db.instantRows(columns, sql"SELECT * FROM my_table"): + ## discard + ## echo columns[0] + ## + ## # Output: + ## # (name: "id", tableName: "my_table", typ: (kind: dbNull, + ## # notNull: false, name: "INTEGER", size: 0, maxReprLen: 0, precision: 0, + ## # scale: 0, min: 0, max: 0, validValues: @[]), primaryKey: false, + ## # foreignKey: false) + ## + ## db.close() var stmt = setupQuery(db, query, args) setColumns(columns, stmt) - while step(stmt) == SQLITE_ROW: - yield stmt - if finalize(stmt) != SQLITE_OK: dbError(db) + try: + while step(stmt) == SQLITE_ROW: + yield stmt + finally: + if finalize(stmt) != SQLITE_OK: dbError(db) proc `[]`*(row: InstantRow, col: int32): string {.inline.} = - ## returns text for given column of the row + ## Returns text for given column of the row. + ## + ## See also: + ## * `instantRows iterator <#instantRows.i,DbConn,SqlQuery,varargs[string,]>`_ + ## example code $column_text(row, col) +proc unsafeColumnAt*(row: InstantRow, index: int32): cstring {.inline.} = + ## Returns cstring for given column of the row. + ## + ## See also: + ## * `instantRows iterator <#instantRows.i,DbConn,SqlQuery,varargs[string,]>`_ + ## example code + column_text(row, index) + proc len*(row: InstantRow): int32 {.inline.} = - ## returns number of columns in the row + ## Returns number of columns in a row. + ## + ## See also: + ## * `instantRows iterator <#instantRows.i,DbConn,SqlQuery,varargs[string,]>`_ + ## example code column_count(row) proc getRow*(db: DbConn, query: SqlQuery, args: varargs[string, `$`]): Row {.tags: [ReadDbEffect].} = - ## retrieves a single row. If the query doesn't return any rows, this proc - ## will return a Row with empty strings for each column. + ## Retrieves a single row. If the query doesn't return any rows, this proc + ## will return a `Row` with empty strings for each column. + ## + ## **Examples:** + ## + ## .. code-block:: Nim + ## + ## let db = open("mytest.db", "", "", "") + ## + ## # Records of my_table: + ## # | id | name | + ## # |----|----------| + ## # | 1 | item#1 | + ## # | 2 | item#2 | + ## + ## doAssert db.getRow(sql"SELECT id, name FROM my_table" + ## ) == Row(@["1", "item#1"]) + ## doAssert db.getRow(sql"SELECT id, name FROM my_table WHERE id = ?", + ## 2) == Row(@["2", "item#2"]) + ## + ## # Returns empty. + ## doAssert db.getRow(sql"INSERT INTO my_table (id, name) VALUES (?, ?)", + ## 3, "item#3") == @[] + ## doAssert db.getRow(sql"DELETE FROM my_table WHERE id = ?", 3) == @[] + ## doAssert db.getRow(sql"UPDATE my_table SET name = 'ITEM#1' WHERE id = ?", + ## 1) == @[] + ## db.close() var stmt = setupQuery(db, query, args) var L = (column_count(stmt)) result = newRow(L)@@ -234,21 +413,77 @@ if finalize(stmt) != SQLITE_OK: dbError(db)
proc getAllRows*(db: DbConn, query: SqlQuery, args: varargs[string, `$`]): seq[Row] {.tags: [ReadDbEffect].} = - ## executes the query and returns the whole result dataset. + ## Executes the query and returns the whole result dataset. + ## + ## **Examples:** + ## + ## .. code-block:: Nim + ## + ## let db = open("mytest.db", "", "", "") + ## + ## # Records of my_table: + ## # | id | name | + ## # |----|----------| + ## # | 1 | item#1 | + ## # | 2 | item#2 | + ## + ## doAssert db.getAllRows(sql"SELECT id, name FROM my_table") == @[Row(@["1", "item#1"]), Row(@["2", "item#2"])] + ## db.close() result = @[] for r in fastRows(db, query, args): result.add(r) iterator rows*(db: DbConn, query: SqlQuery, args: varargs[string, `$`]): Row {.tags: [ReadDbEffect].} = - ## same as `FastRows`, but slower and safe. + ## Similar to `fastRows iterator <#fastRows.i,DbConn,SqlQuery,varargs[string,]>`_, + ## but slower and safe. + ## + ## **Examples:** + ## + ## .. code-block:: Nim + ## + ## let db = open("mytest.db", "", "", "") + ## + ## # Records of my_table: + ## # | id | name | + ## # |----|----------| + ## # | 1 | item#1 | + ## # | 2 | item#2 | + ## + ## for row in db.rows(sql"SELECT id, name FROM my_table"): + ## echo row + ## + ## ## Output: + ## ## @["1", "item#1"] + ## ## @["2", "item#2"] + ## + ## db.close() for r in fastRows(db, query, args): yield r proc getValue*(db: DbConn, query: SqlQuery, args: varargs[string, `$`]): string {.tags: [ReadDbEffect].} = - ## executes the query and returns the first column of the first row of the - ## result dataset. Returns "" if the dataset contains no rows or the database - ## value is NULL. + ## Executes the query and returns the first column of the first row of the + ## result dataset. Returns `""` if the dataset contains no rows or the database + ## value is `NULL`. + ## + ## **Examples:** + ## + ## .. code-block:: Nim + ## + ## let db = open("mytest.db", "", "", "") + ## + ## # Records of my_table: + ## # | id | name | + ## # |----|----------| + ## # | 1 | item#1 | + ## # | 2 | item#2 | + ## + ## doAssert db.getValue(sql"SELECT name FROM my_table WHERE id = ?", + ## 2) == "item#2" + ## doAssert db.getValue(sql"SELECT id, name FROM my_table") == "1" + ## doAssert db.getValue(sql"SELECT name, id FROM my_table") == "item#1" + ## + ## db.close() var stmt = setupQuery(db, query, args) if step(stmt) == SQLITE_ROW: let cb = column_bytes(stmt, 0)@@ -264,10 +499,22 @@
proc tryInsertID*(db: DbConn, query: SqlQuery, args: varargs[string, `$`]): int64 {.tags: [WriteDbEffect], raises: [].} = - ## executes the query (typically "INSERT") and returns the + ## Executes the query (typically "INSERT") and returns the ## generated ID for the row or -1 in case of an error. + ## + ## **Examples:** + ## + ## .. code-block:: Nim + ## + ## let db = open("mytest.db", "", "", "") + ## db.exec(sql"CREATE TABLE my_table (id INTEGER, name VARCHAR(50) NOT NULL)") + ## + ## doAssert db.tryInsertID(sql"INSERT INTO not_exist_table (id, name) VALUES (?, ?)", + ## 1, "item#1") == -1 + ## db.close() + assert(not db.isNil, "Database not connected.") var q = dbFormat(query, args) - var stmt: sqlite3.Pstmt + var stmt: sqlite3.PStmt result = -1 if prepare_v2(db, q, q.len.cint, stmt, nil) == SQLITE_OK: if step(stmt) == SQLITE_DONE:@@ -277,29 +524,86 @@ result = -1
proc insertID*(db: DbConn, query: SqlQuery, args: varargs[string, `$`]): int64 {.tags: [WriteDbEffect].} = - ## executes the query (typically "INSERT") and returns the - ## generated ID for the row. For Postgre this adds - ## ``RETURNING id`` to the query, so it only works if your primary key is - ## named ``id``. + ## Executes the query (typically "INSERT") and returns the + ## generated ID for the row. + ## + ## Raises a `DbError` exception when failed to insert row. + ## For Postgre this adds ``RETURNING id`` to the query, so it only works + ## if your primary key is named ``id``. + ## + ## **Examples:** + ## + ## .. code-block:: Nim + ## + ## let db = open("mytest.db", "", "", "") + ## db.exec(sql"CREATE TABLE my_table (id INTEGER, name VARCHAR(50) NOT NULL)") + ## + ## for i in 0..2: + ## let id = db.insertID(sql"INSERT INTO my_table (id, name) VALUES (?, ?)", i, "item#" & $i) + ## echo "LoopIndex = ", i, ", InsertID = ", id + ## + ## # Output: + ## # LoopIndex = 0, InsertID = 1 + ## # LoopIndex = 1, InsertID = 2 + ## # LoopIndex = 2, InsertID = 3 + ## + ## db.close() result = tryInsertID(db, query, args) if result < 0: dbError(db) proc execAffectedRows*(db: DbConn, query: SqlQuery, args: varargs[string, `$`]): int64 {. tags: [ReadDbEffect, WriteDbEffect].} = - ## executes the query (typically "UPDATE") and returns the + ## Executes the query (typically "UPDATE") and returns the ## number of affected rows. + ## + ## **Examples:** + ## + ## .. code-block:: Nim + ## + ## let db = open("mytest.db", "", "", "") + ## + ## # Records of my_table: + ## # | id | name | + ## # |----|----------| + ## # | 1 | item#1 | + ## # | 2 | item#2 | + ## + ## doAssert db.execAffectedRows(sql"UPDATE my_table SET name = 'TEST'") == 2 + ## + ## db.close() exec(db, query, args) result = changes(db) proc close*(db: DbConn) {.tags: [DbEffect].} = - ## closes the database connection. + ## Closes the database connection. + ## + ## **Examples:** + ## + ## .. code-block:: Nim + ## + ## let db = open("mytest.db", "", "", "") + ## db.close() if sqlite3.close(db) != SQLITE_OK: dbError(db) proc open*(connection, user, password, database: string): DbConn {. tags: [DbEffect].} = - ## opens a database connection. Raises `EDb` if the connection could not - ## be established. Only the ``connection`` parameter is used for ``sqlite``. + ## Opens a database connection. Raises a `DbError` exception if the connection + ## could not be established. + ## + ## **Note:** Only the ``connection`` parameter is used for ``sqlite``. + ## + ## **Examples:** + ## + ## .. code-block:: Nim + ## + ## try: + ## let db = open("mytest.db", "", "", "") + ## ## do something... + ## ## db.getAllRows(sql"SELECT * FROM my_table") + ## db.close() + ## except: + ## stderr.writeLine(getCurrentExceptionMsg()) var db: DbConn if sqlite3.open(connection, db) == SQLITE_OK: result = db@@ -308,10 +612,10 @@ dbError(db)
proc setEncoding*(connection: DbConn, encoding: string): bool {. tags: [DbEffect].} = - ## sets the encoding of a database connection, returns true for - ## success, false for failure. + ## Sets the encoding of a database connection, returns `true` for + ## success, `false` for failure. ## - ## Note that the encoding cannot be changed once it's been set. + ## **Note:** The encoding cannot be changed once it's been set. ## According to SQLite3 documentation, any attempt to change ## the encoding after the database is created will be silently ## ignored.@@ -331,4 +635,4 @@ echo(r[0], r[1])
for r in db.instantRows(sql"select * from tbl1", []): echo(r[0], r[1]) - db_sqlite.close(db) + x_db_sqlite.close(db) # h3rald
@@ -7,8 +7,12 @@ # See the file "copying.txt", included in this
# distribution, for details about the copyright. # -{.deadCodeElim: on.} -# START - removed by h3rald +{.deadCodeElim: on.} # dce option deprecated + +when defined(nimHasStyleChecks): + {.push styleChecks: off.} + +# START - Removed by h3rald #when defined(windows): # when defined(nimOldDlls): # const Lib = "sqlite3.dll"@@ -22,8 +26,13 @@ # Lib = "libsqlite3(|.0).dylib"
#else: # const # Lib = "libsqlite3.so(|.0)" -# END - removed by h3rald -# Also: %s/dynlib: Lib,\?//g +# +#when defined(staticSqlite): +{.pragma: mylib.} +# {.compile: "sqlite3.c".} +#else: +# {.pragma: mylib, dynlib: Lib.} +# END - Removed by h3rald const SQLITE_INTEGER* = 1@@ -55,7 +64,7 @@ SQLITE_PROTOCOL* = 15 # Database lock protocol error
SQLITE_EMPTY* = 16 # Database is empty SQLITE_SCHEMA* = 17 # The database schema changed SQLITE_TOOBIG* = 18 # Too much data for one row of a table - SQLITE_CONSTRAINT* = 19 # Abort due to contraint violation + SQLITE_CONSTRAINT* = 19 # Abort due to constraint violation SQLITE_MISMATCH* = 20 # Data type mismatch SQLITE_MISUSE* = 21 # Library used incorrectly SQLITE_NOLFS* = 22 # Uses OS features not supported on host@@ -105,11 +114,11 @@ PSqlite3* = ptr Sqlite3
PPSqlite3* = ptr PSqlite3 Context{.pure, final.} = object Pcontext* = ptr Context - Tstmt{.pure, final.} = object - Pstmt* = ptr Tstmt + TStmt{.pure, final.} = object + PStmt* = ptr TStmt Value{.pure, final.} = object - Pvalue* = ptr Value - PValueArg* = array[0..127, Pvalue] + PValue* = ptr Value + PValueArg* = array[0..127, PValue] Callback* = proc (para1: pointer, para2: int32, para3, para4: cstringArray): int32{.cdecl.}@@ -124,246 +133,240 @@ Create_collation_func* = proc (para1: pointer, para2: int32, para3: pointer,
para4: int32, para5: pointer): int32{.cdecl.} Collation_needed_func* = proc (para1: pointer, para2: PSqlite3, eTextRep: int32, para4: cstring){.cdecl.} -{.deprecated: [TSqlite3: Sqlite3, TContext: Context, Tvalue: Value, - Tcallback: Callback, Tcreate_function_step_func: Create_function_step_func, - Tcreate_function_func_func: Create_function_func_func, - Tcreate_function_final_func: Create_function_final_func, - Tresult_func: Result_func, Tcreate_collation_func: Create_collation_func, - Tcollation_needed_func: Collation_needed_func].} const SQLITE_STATIC* = nil SQLITE_TRANSIENT* = cast[Tbind_destructor_func](-1) -proc close*(para1: PSqlite3): int32{.cdecl, importc: "sqlite3_close".} +proc close*(para1: PSqlite3): int32{.cdecl, mylib, importc: "sqlite3_close".} proc exec*(para1: PSqlite3, sql: cstring, para3: Callback, para4: pointer, - errmsg: var cstring): int32{.cdecl, + errmsg: var cstring): int32{.cdecl, mylib, importc: "sqlite3_exec".} -proc last_insert_rowid*(para1: PSqlite3): int64{.cdecl, +proc last_insert_rowid*(para1: PSqlite3): int64{.cdecl, mylib, importc: "sqlite3_last_insert_rowid".} -proc changes*(para1: PSqlite3): int32{.cdecl, importc: "sqlite3_changes".} -proc total_changes*(para1: PSqlite3): int32{.cdecl, +proc changes*(para1: PSqlite3): int32{.cdecl, mylib, importc: "sqlite3_changes".} +proc total_changes*(para1: PSqlite3): int32{.cdecl, mylib, importc: "sqlite3_total_changes".} -proc interrupt*(para1: PSqlite3){.cdecl, importc: "sqlite3_interrupt".} -proc complete*(sql: cstring): int32{.cdecl, +proc interrupt*(para1: PSqlite3){.cdecl, mylib, importc: "sqlite3_interrupt".} +proc complete*(sql: cstring): int32{.cdecl, mylib, importc: "sqlite3_complete".} -proc complete16*(sql: pointer): int32{.cdecl, +proc complete16*(sql: pointer): int32{.cdecl, mylib, importc: "sqlite3_complete16".} proc busy_handler*(para1: PSqlite3, para2: proc (para1: pointer, para2: int32): int32{.cdecl.}, - para3: pointer): int32{.cdecl, + para3: pointer): int32{.cdecl, mylib, importc: "sqlite3_busy_handler".} -proc busy_timeout*(para1: PSqlite3, ms: int32): int32{.cdecl, +proc busy_timeout*(para1: PSqlite3, ms: int32): int32{.cdecl, mylib, importc: "sqlite3_busy_timeout".} proc get_table*(para1: PSqlite3, sql: cstring, resultp: var cstringArray, nrow, ncolumn: var cint, errmsg: ptr cstring): int32{.cdecl, - importc: "sqlite3_get_table".} -proc free_table*(result: cstringArray){.cdecl, + mylib, importc: "sqlite3_get_table".} +proc free_table*(result: cstringArray){.cdecl, mylib, importc: "sqlite3_free_table".} # Todo: see how translate sqlite3_mprintf, sqlite3_vmprintf, sqlite3_snprintf # function sqlite3_mprintf(_para1:Pchar; args:array of const):Pchar;cdecl; external Sqlite3Lib name 'sqlite3_mprintf'; -proc mprintf*(para1: cstring): cstring{.cdecl, varargs, +proc mprintf*(para1: cstring): cstring{.cdecl, varargs, mylib, importc: "sqlite3_mprintf".} #function sqlite3_vmprintf(_para1:Pchar; _para2:va_list):Pchar;cdecl; external Sqlite3Lib name 'sqlite3_vmprintf'; -proc free*(z: cstring){.cdecl, importc: "sqlite3_free".} +proc free*(z: cstring){.cdecl, mylib, importc: "sqlite3_free".} #function sqlite3_snprintf(_para1:longint; _para2:Pchar; _para3:Pchar; args:array of const):Pchar;cdecl; external Sqlite3Lib name 'sqlite3_snprintf'; proc snprintf*(para1: int32, para2: cstring, para3: cstring): cstring{.cdecl, - varargs, importc: "sqlite3_snprintf".} + mylib, varargs, importc: "sqlite3_snprintf".} proc set_authorizer*(para1: PSqlite3, xAuth: proc (para1: pointer, para2: int32, para3: cstring, para4: cstring, para5: cstring, para6: cstring): int32{. - cdecl.}, pUserData: pointer): int32{.cdecl, + cdecl.}, pUserData: pointer): int32{.cdecl, mylib, importc: "sqlite3_set_authorizer".} proc trace*(para1: PSqlite3, xTrace: proc (para1: pointer, para2: cstring){.cdecl.}, - para3: pointer): pointer{.cdecl, + para3: pointer): pointer{.cdecl, mylib, importc: "sqlite3_trace".} proc progress_handler*(para1: PSqlite3, para2: int32, para3: proc (para1: pointer): int32{.cdecl.}, - para4: pointer){.cdecl, + para4: pointer){.cdecl, mylib, importc: "sqlite3_progress_handler".} proc commit_hook*(para1: PSqlite3, para2: proc (para1: pointer): int32{.cdecl.}, - para3: pointer): pointer{.cdecl, + para3: pointer): pointer{.cdecl, mylib, importc: "sqlite3_commit_hook".} -proc open*(filename: cstring, ppDb: var PSqlite3): int32{.cdecl, +proc open*(filename: cstring, ppDb: var PSqlite3): int32{.cdecl, mylib, importc: "sqlite3_open".} -proc open16*(filename: pointer, ppDb: var PSqlite3): int32{.cdecl, +proc open16*(filename: pointer, ppDb: var PSqlite3): int32{.cdecl, mylib, importc: "sqlite3_open16".} -proc errcode*(db: PSqlite3): int32{.cdecl, importc: "sqlite3_errcode".} -proc errmsg*(para1: PSqlite3): cstring{.cdecl, importc: "sqlite3_errmsg".} -proc errmsg16*(para1: PSqlite3): pointer{.cdecl, +proc errcode*(db: PSqlite3): int32{.cdecl, mylib, importc: "sqlite3_errcode".} +proc errmsg*(para1: PSqlite3): cstring{.cdecl, mylib, importc: "sqlite3_errmsg".} +proc errmsg16*(para1: PSqlite3): pointer{.cdecl, mylib, importc: "sqlite3_errmsg16".} -proc prepare*(db: PSqlite3, zSql: cstring, nBytes: int32, ppStmt: var Pstmt, - pzTail: ptr cstring): int32{.cdecl, +proc prepare*(db: PSqlite3, zSql: cstring, nBytes: int32, ppStmt: var PStmt, + pzTail: ptr cstring): int32{.cdecl, mylib, importc: "sqlite3_prepare".} -proc prepare_v2*(db: PSqlite3, zSql: cstring, nByte: cint, ppStmt: var Pstmt, +proc prepare_v2*(db: PSqlite3, zSql: cstring, nByte: cint, ppStmt: var PStmt, pzTail: ptr cstring): cint {. - importc: "sqlite3_prepare_v2", cdecl, .} + importc: "sqlite3_prepare_v2", cdecl, mylib.} -proc prepare16*(db: PSqlite3, zSql: pointer, nBytes: int32, ppStmt: var Pstmt, - pzTail: var pointer): int32{.cdecl, +proc prepare16*(db: PSqlite3, zSql: pointer, nBytes: int32, ppStmt: var PStmt, + pzTail: var pointer): int32{.cdecl, mylib, importc: "sqlite3_prepare16".} -proc bind_blob*(para1: Pstmt, para2: int32, para3: pointer, n: int32, - para5: Tbind_destructor_func): int32{.cdecl, +proc bind_blob*(para1: PStmt, para2: int32, para3: pointer, n: int32, + para5: Tbind_destructor_func): int32{.cdecl, mylib, importc: "sqlite3_bind_blob".} -proc bind_double*(para1: Pstmt, para2: int32, para3: float64): int32{.cdecl, - importc: "sqlite3_bind_double".} -proc bind_int*(para1: Pstmt, para2: int32, para3: int32): int32{.cdecl, - importc: "sqlite3_bind_int".} -proc bind_int64*(para1: Pstmt, para2: int32, para3: int64): int32{.cdecl, - importc: "sqlite3_bind_int64".} -proc bind_null*(para1: Pstmt, para2: int32): int32{.cdecl, +proc bind_double*(para1: PStmt, para2: int32, para3: float64): int32{.cdecl, + mylib, importc: "sqlite3_bind_double".} +proc bind_int*(para1: PStmt, para2: int32, para3: int32): int32{.cdecl, + mylib, importc: "sqlite3_bind_int".} +proc bind_int64*(para1: PStmt, para2: int32, para3: int64): int32{.cdecl, + mylib, importc: "sqlite3_bind_int64".} +proc bind_null*(para1: PStmt, para2: int32): int32{.cdecl, mylib, importc: "sqlite3_bind_null".} -proc bind_text*(para1: Pstmt, para2: int32, para3: cstring, n: int32, - para5: Tbind_destructor_func): int32{.cdecl, +proc bind_text*(para1: PStmt, para2: int32, para3: cstring, n: int32, + para5: Tbind_destructor_func): int32{.cdecl, mylib, importc: "sqlite3_bind_text".} -proc bind_text16*(para1: Pstmt, para2: int32, para3: pointer, para4: int32, - para5: Tbind_destructor_func): int32{.cdecl, +proc bind_text16*(para1: PStmt, para2: int32, para3: pointer, para4: int32, + para5: Tbind_destructor_func): int32{.cdecl, mylib, importc: "sqlite3_bind_text16".} #function sqlite3_bind_value(_para1:Psqlite3_stmt; _para2:longint; _para3:Psqlite3_value):longint;cdecl; external Sqlite3Lib name 'sqlite3_bind_value'; #These overloaded functions were introduced to allow the use of SQLITE_STATIC and SQLITE_TRANSIENT #It's the c world man ;-) -proc bind_blob*(para1: Pstmt, para2: int32, para3: pointer, n: int32, - para5: int32): int32{.cdecl, +proc bind_blob*(para1: PStmt, para2: int32, para3: pointer, n: int32, + para5: int32): int32{.cdecl, mylib, importc: "sqlite3_bind_blob".} -proc bind_text*(para1: Pstmt, para2: int32, para3: cstring, n: int32, - para5: int32): int32{.cdecl, +proc bind_text*(para1: PStmt, para2: int32, para3: cstring, n: int32, + para5: int32): int32{.cdecl, mylib, importc: "sqlite3_bind_text".} -proc bind_text16*(para1: Pstmt, para2: int32, para3: pointer, para4: int32, - para5: int32): int32{.cdecl, +proc bind_text16*(para1: PStmt, para2: int32, para3: pointer, para4: int32, + para5: int32): int32{.cdecl, mylib, importc: "sqlite3_bind_text16".} -proc bind_parameter_count*(para1: Pstmt): int32{.cdecl, +proc bind_parameter_count*(para1: PStmt): int32{.cdecl, mylib, importc: "sqlite3_bind_parameter_count".} -proc bind_parameter_name*(para1: Pstmt, para2: int32): cstring{.cdecl, - importc: "sqlite3_bind_parameter_name".} -proc bind_parameter_index*(para1: Pstmt, zName: cstring): int32{.cdecl, - importc: "sqlite3_bind_parameter_index".} -proc clear_bindings*(para1: Pstmt): int32 {.cdecl, - importc: "sqlite3_clear_bindings".} -proc column_count*(pStmt: Pstmt): int32{.cdecl, +proc bind_parameter_name*(para1: PStmt, para2: int32): cstring{.cdecl, + mylib, importc: "sqlite3_bind_parameter_name".} +proc bind_parameter_index*(para1: PStmt, zName: cstring): int32{.cdecl, + mylib, importc: "sqlite3_bind_parameter_index".} +proc clear_bindings*(para1: PStmt): int32 {.cdecl, + mylib, importc: "sqlite3_clear_bindings".} +proc column_count*(PStmt: PStmt): int32{.cdecl, mylib, importc: "sqlite3_column_count".} -proc column_name*(para1: Pstmt, para2: int32): cstring{.cdecl, +proc column_name*(para1: PStmt, para2: int32): cstring{.cdecl, mylib, importc: "sqlite3_column_name".} -proc column_table_name*(para1: Pstmt; para2: int32): cstring{.cdecl, +proc column_table_name*(para1: PStmt; para2: int32): cstring{.cdecl, mylib, importc: "sqlite3_column_table_name".} -proc column_name16*(para1: Pstmt, para2: int32): pointer{.cdecl, +proc column_name16*(para1: PStmt, para2: int32): pointer{.cdecl, mylib, importc: "sqlite3_column_name16".} -proc column_decltype*(para1: Pstmt, i: int32): cstring{.cdecl, +proc column_decltype*(para1: PStmt, i: int32): cstring{.cdecl, mylib, importc: "sqlite3_column_decltype".} -proc column_decltype16*(para1: Pstmt, para2: int32): pointer{.cdecl, - importc: "sqlite3_column_decltype16".} -proc step*(para1: Pstmt): int32{.cdecl, importc: "sqlite3_step".} -proc data_count*(pStmt: Pstmt): int32{.cdecl, +proc column_decltype16*(para1: PStmt, para2: int32): pointer{.cdecl, + mylib, importc: "sqlite3_column_decltype16".} +proc step*(para1: PStmt): int32{.cdecl, mylib, importc: "sqlite3_step".} +proc data_count*(PStmt: PStmt): int32{.cdecl, mylib, importc: "sqlite3_data_count".} -proc column_blob*(para1: Pstmt, iCol: int32): pointer{.cdecl, +proc column_blob*(para1: PStmt, iCol: int32): pointer{.cdecl, mylib, importc: "sqlite3_column_blob".} -proc column_bytes*(para1: Pstmt, iCol: int32): int32{.cdecl, +proc column_bytes*(para1: PStmt, iCol: int32): int32{.cdecl, mylib, importc: "sqlite3_column_bytes".} -proc column_bytes16*(para1: Pstmt, iCol: int32): int32{.cdecl, +proc column_bytes16*(para1: PStmt, iCol: int32): int32{.cdecl, mylib, importc: "sqlite3_column_bytes16".} -proc column_double*(para1: Pstmt, iCol: int32): float64{.cdecl, +proc column_double*(para1: PStmt, iCol: int32): float64{.cdecl, mylib, importc: "sqlite3_column_double".} -proc column_int*(para1: Pstmt, iCol: int32): int32{.cdecl, +proc column_int*(para1: PStmt, iCol: int32): int32{.cdecl, mylib, importc: "sqlite3_column_int".} -proc column_int64*(para1: Pstmt, iCol: int32): int64{.cdecl, +proc column_int64*(para1: PStmt, iCol: int32): int64{.cdecl, mylib, importc: "sqlite3_column_int64".} -proc column_text*(para1: Pstmt, iCol: int32): cstring{.cdecl, +proc column_text*(para1: PStmt, iCol: int32): cstring{.cdecl, mylib, importc: "sqlite3_column_text".} -proc column_text16*(para1: Pstmt, iCol: int32): pointer{.cdecl, +proc column_text16*(para1: PStmt, iCol: int32): pointer{.cdecl, mylib, importc: "sqlite3_column_text16".} -proc column_type*(para1: Pstmt, iCol: int32): int32{.cdecl, +proc column_type*(para1: PStmt, iCol: int32): int32{.cdecl, mylib, importc: "sqlite3_column_type".} -proc finalize*(pStmt: Pstmt): int32{.cdecl, +proc finalize*(PStmt: PStmt): int32{.cdecl, mylib, importc: "sqlite3_finalize".} -proc reset*(pStmt: Pstmt): int32{.cdecl, importc: "sqlite3_reset".} +proc reset*(PStmt: PStmt): int32{.cdecl, mylib, importc: "sqlite3_reset".} proc create_function*(para1: PSqlite3, zFunctionName: cstring, nArg: int32, eTextRep: int32, para5: pointer, xFunc: Create_function_func_func, xStep: Create_function_step_func, xFinal: Create_function_final_func): int32{.cdecl, - importc: "sqlite3_create_function".} + mylib, importc: "sqlite3_create_function".} proc create_function16*(para1: PSqlite3, zFunctionName: pointer, nArg: int32, eTextRep: int32, para5: pointer, xFunc: Create_function_func_func, xStep: Create_function_step_func, xFinal: Create_function_final_func): int32{.cdecl, - importc: "sqlite3_create_function16".} -proc aggregate_count*(para1: Pcontext): int32{.cdecl, + mylib, importc: "sqlite3_create_function16".} +proc aggregate_count*(para1: Pcontext): int32{.cdecl, mylib, importc: "sqlite3_aggregate_count".} -proc value_blob*(para1: Pvalue): pointer{.cdecl, +proc value_blob*(para1: PValue): pointer{.cdecl, mylib, importc: "sqlite3_value_blob".} -proc value_bytes*(para1: Pvalue): int32{.cdecl, +proc value_bytes*(para1: PValue): int32{.cdecl, mylib, importc: "sqlite3_value_bytes".} -proc value_bytes16*(para1: Pvalue): int32{.cdecl, +proc value_bytes16*(para1: PValue): int32{.cdecl, mylib, importc: "sqlite3_value_bytes16".} -proc value_double*(para1: Pvalue): float64{.cdecl, +proc value_double*(para1: PValue): float64{.cdecl, mylib, importc: "sqlite3_value_double".} -proc value_int*(para1: Pvalue): int32{.cdecl, +proc value_int*(para1: PValue): int32{.cdecl, mylib, importc: "sqlite3_value_int".} -proc value_int64*(para1: Pvalue): int64{.cdecl, +proc value_int64*(para1: PValue): int64{.cdecl, mylib, importc: "sqlite3_value_int64".} -proc value_text*(para1: Pvalue): cstring{.cdecl, +proc value_text*(para1: PValue): cstring{.cdecl, mylib, importc: "sqlite3_value_text".} -proc value_text16*(para1: Pvalue): pointer{.cdecl, +proc value_text16*(para1: PValue): pointer{.cdecl, mylib, importc: "sqlite3_value_text16".} -proc value_text16le*(para1: Pvalue): pointer{.cdecl, +proc value_text16le*(para1: PValue): pointer{.cdecl, mylib, importc: "sqlite3_value_text16le".} -proc value_text16be*(para1: Pvalue): pointer{.cdecl, +proc value_text16be*(para1: PValue): pointer{.cdecl, mylib, importc: "sqlite3_value_text16be".} -proc value_type*(para1: Pvalue): int32{.cdecl, +proc value_type*(para1: PValue): int32{.cdecl, mylib, importc: "sqlite3_value_type".} proc aggregate_context*(para1: Pcontext, nBytes: int32): pointer{.cdecl, - importc: "sqlite3_aggregate_context".} -proc user_data*(para1: Pcontext): pointer{.cdecl, + mylib, importc: "sqlite3_aggregate_context".} +proc user_data*(para1: Pcontext): pointer{.cdecl, mylib, importc: "sqlite3_user_data".} -proc get_auxdata*(para1: Pcontext, para2: int32): pointer{.cdecl, +proc get_auxdata*(para1: Pcontext, para2: int32): pointer{.cdecl, mylib, importc: "sqlite3_get_auxdata".} proc set_auxdata*(para1: Pcontext, para2: int32, para3: pointer, - para4: proc (para1: pointer){.cdecl.}){.cdecl, + para4: proc (para1: pointer){.cdecl.}){.cdecl, mylib, importc: "sqlite3_set_auxdata".} proc result_blob*(para1: Pcontext, para2: pointer, para3: int32, - para4: Result_func){.cdecl, + para4: Result_func){.cdecl, mylib, importc: "sqlite3_result_blob".} -proc result_double*(para1: Pcontext, para2: float64){.cdecl, +proc result_double*(para1: Pcontext, para2: float64){.cdecl, mylib, importc: "sqlite3_result_double".} proc result_error*(para1: Pcontext, para2: cstring, para3: int32){.cdecl, - importc: "sqlite3_result_error".} + mylib, importc: "sqlite3_result_error".} proc result_error16*(para1: Pcontext, para2: pointer, para3: int32){.cdecl, - importc: "sqlite3_result_error16".} -proc result_int*(para1: Pcontext, para2: int32){.cdecl, + mylib, importc: "sqlite3_result_error16".} +proc result_int*(para1: Pcontext, para2: int32){.cdecl, mylib, importc: "sqlite3_result_int".} -proc result_int64*(para1: Pcontext, para2: int64){.cdecl, +proc result_int64*(para1: Pcontext, para2: int64){.cdecl, mylib, importc: "sqlite3_result_int64".} -proc result_null*(para1: Pcontext){.cdecl, +proc result_null*(para1: Pcontext){.cdecl, mylib, importc: "sqlite3_result_null".} proc result_text*(para1: Pcontext, para2: cstring, para3: int32, - para4: Result_func){.cdecl, + para4: Result_func){.cdecl, mylib, importc: "sqlite3_result_text".} proc result_text16*(para1: Pcontext, para2: pointer, para3: int32, - para4: Result_func){.cdecl, + para4: Result_func){.cdecl, mylib, importc: "sqlite3_result_text16".} proc result_text16le*(para1: Pcontext, para2: pointer, para3: int32, - para4: Result_func){.cdecl, + para4: Result_func){.cdecl, mylib, importc: "sqlite3_result_text16le".} proc result_text16be*(para1: Pcontext, para2: pointer, para3: int32, - para4: Result_func){.cdecl, + para4: Result_func){.cdecl, mylib, importc: "sqlite3_result_text16be".} -proc result_value*(para1: Pcontext, para2: Pvalue){.cdecl, +proc result_value*(para1: Pcontext, para2: PValue){.cdecl, mylib, importc: "sqlite3_result_value".} proc create_collation*(para1: PSqlite3, zName: cstring, eTextRep: int32, para4: pointer, xCompare: Create_collation_func): int32{. - cdecl, importc: "sqlite3_create_collation".} + cdecl, mylib, importc: "sqlite3_create_collation".} proc create_collation16*(para1: PSqlite3, zName: cstring, eTextRep: int32, para4: pointer, xCompare: Create_collation_func): int32{. - cdecl, importc: "sqlite3_create_collation16".} + cdecl, mylib, importc: "sqlite3_create_collation16".} proc collation_needed*(para1: PSqlite3, para2: pointer, para3: Collation_needed_func): int32{. - cdecl, importc: "sqlite3_collation_needed".} + cdecl, mylib, importc: "sqlite3_collation_needed".} proc collation_needed16*(para1: PSqlite3, para2: pointer, para3: Collation_needed_func): int32{. - cdecl, importc: "sqlite3_collation_needed16".} -proc libversion*(): cstring{.cdecl, importc: "sqlite3_libversion".} + cdecl, mylib, importc: "sqlite3_collation_needed16".} +proc libversion*(): cstring{.cdecl, mylib, importc: "sqlite3_libversion".} #Alias for allowing better code portability (win32 is not working with external variables) -proc version*(): cstring{.cdecl, importc: "sqlite3_libversion".} +proc version*(): cstring{.cdecl, mylib, importc: "sqlite3_libversion".} # Not published functions -proc libversion_number*(): int32{.cdecl, +proc libversion_number*(): int32{.cdecl, mylib, importc: "sqlite3_libversion_number".} #function sqlite3_key(db:Psqlite3; pKey:pointer; nKey:longint):longint;cdecl; external Sqlite3Lib name 'sqlite3_key'; #function sqlite3_rekey(db:Psqlite3; pKey:pointer; nKey:longint):longint;cdecl; external Sqlite3Lib name 'sqlite3_rekey';@@ -371,3 +374,6 @@ #function sqlite3_sleep(_para1:longint):longint;cdecl; external Sqlite3Lib name 'sqlite3_sleep';
#function sqlite3_expired(_para1:Psqlite3_stmt):longint;cdecl; external Sqlite3Lib name 'sqlite3_expired'; #function sqlite3_global_recover:longint;cdecl; external Sqlite3Lib name 'sqlite3_global_recover'; # implementation + +when defined(nimHasStyleChecks): + {.pop.}
@@ -1,6 +1,6 @@
/****************************************************************************** ** This file is an amalgamation of many separate C source files from SQLite -** version 3.25.1. By combining all the individual C code files into this +** version 3.29.0. By combining all the individual C code files into this ** single large file, the entire code can be compiled as a single translation ** unit. This allows many compilers to do optimizations that would not be ** possible if the files were compiled separately. Performance improvements@@ -39,7 +39,7 @@ ** This file implements routines used to report what compile-time options
** SQLite was built with. */ -#ifndef SQLITE_OMIT_COMPILEOPTION_DIAGS +#ifndef SQLITE_OMIT_COMPILEOPTION_DIAGS /* IMP: R-16824-07538 */ /* ** Include the configuration header output by 'configure' if we're using the@@ -260,6 +260,9 @@ #endif
#if SQLITE_ENABLE_FTS5 "ENABLE_FTS5", #endif +#if SQLITE_ENABLE_GEOPOLY + "ENABLE_GEOPOLY", +#endif #if SQLITE_ENABLE_HIDDEN_COLUMNS "ENABLE_HIDDEN_COLUMNS", #endif@@ -289,6 +292,9 @@ "ENABLE_MEMSYS5",
#endif #if SQLITE_ENABLE_MULTIPLEX "ENABLE_MULTIPLEX", +#endif +#if SQLITE_ENABLE_NORMALIZE + "ENABLE_NORMALIZE", #endif #if SQLITE_ENABLE_NULL_TRIM "ENABLE_NULL_TRIM",@@ -882,6 +888,11 @@ #pragma warning(disable : 4702)
#pragma warning(disable : 4706) #endif /* defined(_MSC_VER) */ +#if defined(_MSC_VER) && !defined(_WIN64) +#undef SQLITE_4_BYTE_ALIGNED_MALLOC +#define SQLITE_4_BYTE_ALIGNED_MALLOC +#endif /* defined(_MSC_VER) && !defined(_WIN64) */ + #endif /* SQLITE_MSVC_H */ /************** End of msvc.h ************************************************/@@ -1156,9 +1167,9 @@ ** See also: [sqlite3_libversion()],
** [sqlite3_libversion_number()], [sqlite3_sourceid()], ** [sqlite_version()] and [sqlite_source_id()]. */ -#define SQLITE_VERSION "3.25.1" -#define SQLITE_VERSION_NUMBER 3025001 -#define SQLITE_SOURCE_ID "2018-09-18 20:20:44 2ac9003de44da7dafa3fbb1915ac5725a9275c86bf2f3b7aa19321bf1460b386" +#define SQLITE_VERSION "3.29.0" +#define SQLITE_VERSION_NUMBER 3029000 +#define SQLITE_SOURCE_ID "2019-07-10 17:32:03 fc82b73eaac8b36950e527f12c4b5dc1e147e6f4ad2217ae43ad82882a88bfa6" /* ** CAPI3REF: Run-Time Library Version Numbers@@ -1222,6 +1233,9 @@ */
#ifndef SQLITE_OMIT_COMPILEOPTION_DIAGS SQLITE_API int sqlite3_compileoption_used(const char *zOptName); SQLITE_API const char *sqlite3_compileoption_get(int N); +#else +# define sqlite3_compileoption_used(X) 0 +# define sqlite3_compileoption_get(X) ((void*)0) #endif /*@@ -1856,6 +1870,15 @@ ** is often close. The underlying VFS might choose to preallocate database
** file space based on this hint in order to help writes to the database ** file run faster. ** +** <li>[[SQLITE_FCNTL_SIZE_LIMIT]] +** The [SQLITE_FCNTL_SIZE_LIMIT] opcode is used by in-memory VFS that +** implements [sqlite3_deserialize()] to set an upper bound on the size +** of the in-memory database. The argument is a pointer to a [sqlite3_int64]. +** If the integer pointed to is negative, then it is filled in with the +** current limit. Otherwise the limit is set to the larger of the value +** of the integer pointed to and the current database size. The integer +** pointed to is set to the new limit. +** ** <li>[[SQLITE_FCNTL_CHUNK_SIZE]] ** The [SQLITE_FCNTL_CHUNK_SIZE] opcode is used to request that the VFS ** extends and truncates the database file in chunks of a size specified@@ -2164,6 +2187,7 @@ #define SQLITE_FCNTL_COMMIT_ATOMIC_WRITE 32
#define SQLITE_FCNTL_ROLLBACK_ATOMIC_WRITE 33 #define SQLITE_FCNTL_LOCK_TIMEOUT 34 #define SQLITE_FCNTL_DATA_VERSION 35 +#define SQLITE_FCNTL_SIZE_LIMIT 36 /* deprecated names */ #define SQLITE_GET_LOCKPROXYFILE SQLITE_FCNTL_GET_LOCKPROXYFILE@@ -2316,8 +2340,14 @@ ** [[sqlite3_vfs.xAccess]]
** ^The flags argument to xAccess() may be [SQLITE_ACCESS_EXISTS] ** to test for the existence of a file, or [SQLITE_ACCESS_READWRITE] to ** test whether a file is readable and writable, or [SQLITE_ACCESS_READ] -** to test whether a file is at least readable. The file can be a -** directory. +** to test whether a file is at least readable. The SQLITE_ACCESS_READ +** flag is never actually used and is not implemented in the built-in +** VFSes of SQLite. The file is named by the second argument and can be a +** directory. The xAccess method returns [SQLITE_OK] on success or some +** non-zero error code if there is an I/O error or if the name of +** the file given in the second argument is illegal. If SQLITE_OK +** is returned, then non-zero or zero is written into *pResOut to indicate +** whether or not the file is accessible. ** ** ^SQLite will always allocate at least mxPathname+1 bytes for the ** output buffer xFullPathname. The exact size of the output buffer@@ -3005,6 +3035,17 @@ ** value for this option is to never use this optimization. Specifying a
** negative value for this option restores the default behaviour. ** This option is only available if SQLite is compiled with the ** [SQLITE_ENABLE_SORTER_REFERENCES] compile-time option. +** +** [[SQLITE_CONFIG_MEMDB_MAXSIZE]] +** <dt>SQLITE_CONFIG_MEMDB_MAXSIZE +** <dd>The SQLITE_CONFIG_MEMDB_MAXSIZE option accepts a single parameter +** [sqlite3_int64] parameter which is the default maximum size for an in-memory +** database created using [sqlite3_deserialize()]. This default maximum +** size can be adjusted up or down for individual databases using the +** [SQLITE_FCNTL_SIZE_LIMIT] [sqlite3_file_control|file-control]. If this +** configuration setting is never used, then the default maximum is determined +** by the [SQLITE_MEMDB_DEFAULT_MAXSIZE] compile-time option. If that +** compile-time option is not set, then the default maximum is 1073741824. ** </dl> */ #define SQLITE_CONFIG_SINGLETHREAD 1 /* nil */@@ -3035,6 +3076,7 @@ #define SQLITE_CONFIG_PMASZ 25 /* unsigned int szPma */
#define SQLITE_CONFIG_STMTJRNL_SPILL 26 /* int nByte */ #define SQLITE_CONFIG_SMALL_MALLOC 27 /* boolean */ #define SQLITE_CONFIG_SORTERREF_SIZE 28 /* int nByte */ +#define SQLITE_CONFIG_MEMDB_MAXSIZE 29 /* sqlite3_int64 */ /* ** CAPI3REF: Database Connection Configuration Options@@ -3050,6 +3092,7 @@ ** non-zero [error code] if a discontinued or unsupported configuration option
** is invoked. ** ** <dl> +** [[SQLITE_DBCONFIG_LOOKASIDE]] ** <dt>SQLITE_DBCONFIG_LOOKASIDE</dt> ** <dd> ^This option takes three additional arguments that determine the ** [lookaside memory allocator] configuration for the [database connection].@@ -3072,6 +3115,7 @@ ** Any attempt to change the lookaside memory configuration when lookaside
** memory is in use leaves the configuration unchanged and returns ** [SQLITE_BUSY].)^</dd> ** +** [[SQLITE_DBCONFIG_ENABLE_FKEY]] ** <dt>SQLITE_DBCONFIG_ENABLE_FKEY</dt> ** <dd> ^This option is used to enable or disable the enforcement of ** [foreign key constraints]. There should be two additional arguments.@@ -3082,6 +3126,7 @@ ** is written 0 or 1 to indicate whether FK enforcement is off or on
** following this call. The second parameter may be a NULL pointer, in ** which case the FK enforcement setting is not reported back. </dd> ** +** [[SQLITE_DBCONFIG_ENABLE_TRIGGER]] ** <dt>SQLITE_DBCONFIG_ENABLE_TRIGGER</dt> ** <dd> ^This option is used to enable or disable [CREATE TRIGGER | triggers]. ** There should be two additional arguments.@@ -3092,9 +3137,10 @@ ** is written 0 or 1 to indicate whether triggers are disabled or enabled
** following this call. The second parameter may be a NULL pointer, in ** which case the trigger setting is not reported back. </dd> ** +** [[SQLITE_DBCONFIG_ENABLE_FTS3_TOKENIZER]] ** <dt>SQLITE_DBCONFIG_ENABLE_FTS3_TOKENIZER</dt> -** <dd> ^This option is used to enable or disable the two-argument -** version of the [fts3_tokenizer()] function which is part of the +** <dd> ^This option is used to enable or disable the +** [fts3_tokenizer()] function which is part of the ** [FTS3] full-text search engine extension. ** There should be two additional arguments. ** The first argument is an integer which is 0 to disable fts3_tokenizer() or@@ -3105,6 +3151,7 @@ ** is written 0 or 1 to indicate whether fts3_tokenizer is disabled or enabled
** following this call. The second parameter may be a NULL pointer, in ** which case the new setting is not reported back. </dd> ** +** [[SQLITE_DBCONFIG_ENABLE_LOAD_EXTENSION]] ** <dt>SQLITE_DBCONFIG_ENABLE_LOAD_EXTENSION</dt> ** <dd> ^This option is used to enable or disable the [sqlite3_load_extension()] ** interface independently of the [load_extension()] SQL function.@@ -3122,7 +3169,7 @@ ** is disabled or enabled following this call. The second parameter may
** be a NULL pointer, in which case the new setting is not reported back. ** </dd> ** -** <dt>SQLITE_DBCONFIG_MAINDBNAME</dt> +** [[SQLITE_DBCONFIG_MAINDBNAME]] <dt>SQLITE_DBCONFIG_MAINDBNAME</dt> ** <dd> ^This option is used to change the name of the "main" database ** schema. ^The sole argument is a pointer to a constant UTF8 string ** which will become the new schema name in place of "main". ^SQLite@@ -3131,6 +3178,7 @@ ** must ensure that the argument passed into this DBCONFIG option is unchanged
** until after the database connection closes. ** </dd> ** +** [[SQLITE_DBCONFIG_NO_CKPT_ON_CLOSE]] ** <dt>SQLITE_DBCONFIG_NO_CKPT_ON_CLOSE</dt> ** <dd> Usually, when a database in wal mode is closed or detached from a ** database handle, SQLite checks if this will mean that there are now no@@ -3144,7 +3192,7 @@ ** into which is written 0 or 1 to indicate whether checkpoints-on-close
** have been disabled - 0 if they are not disabled, 1 if they are. ** </dd> ** -** <dt>SQLITE_DBCONFIG_ENABLE_QPSG</dt> +** [[SQLITE_DBCONFIG_ENABLE_QPSG]] <dt>SQLITE_DBCONFIG_ENABLE_QPSG</dt> ** <dd>^(The SQLITE_DBCONFIG_ENABLE_QPSG option activates or deactivates ** the [query planner stability guarantee] (QPSG). When the QPSG is active, ** a single SQL query statement will always use the same algorithm regardless@@ -3160,7 +3208,7 @@ ** is written 0 or 1 to indicate whether the QPSG is disabled or enabled
** following this call. ** </dd> ** -** <dt>SQLITE_DBCONFIG_TRIGGER_EQP</dt> +** [[SQLITE_DBCONFIG_TRIGGER_EQP]] <dt>SQLITE_DBCONFIG_TRIGGER_EQP</dt> ** <dd> By default, the output of EXPLAIN QUERY PLAN commands does not ** include output for any operations performed by trigger programs. This ** option is used to set or clear (the default) a flag that governs this@@ -3172,7 +3220,7 @@ ** 0 or 1 to indicate whether output-for-triggers has been disabled - 0 if
** it is not disabled, 1 if it is. ** </dd> ** -** <dt>SQLITE_DBCONFIG_RESET_DATABASE</dt> +** [[SQLITE_DBCONFIG_RESET_DATABASE]] <dt>SQLITE_DBCONFIG_RESET_DATABASE</dt> ** <dd> Set the SQLITE_DBCONFIG_RESET_DATABASE flag and then run ** [VACUUM] in order to reset a database back to an empty database ** with no schema and no content. The following process works even for@@ -3191,6 +3239,58 @@ ** </ol>
** Because resetting a database is destructive and irreversible, the ** process requires the use of this obscure API and multiple steps to help ** ensure that it does not happen by accident. +** +** [[SQLITE_DBCONFIG_DEFENSIVE]] <dt>SQLITE_DBCONFIG_DEFENSIVE</dt> +** <dd>The SQLITE_DBCONFIG_DEFENSIVE option activates or deactivates the +** "defensive" flag for a database connection. When the defensive +** flag is enabled, language features that allow ordinary SQL to +** deliberately corrupt the database file are disabled. The disabled +** features include but are not limited to the following: +** <ul> +** <li> The [PRAGMA writable_schema=ON] statement. +** <li> The [PRAGMA journal_mode=OFF] statement. +** <li> Writes to the [sqlite_dbpage] virtual table. +** <li> Direct writes to [shadow tables]. +** </ul> +** </dd> +** +** [[SQLITE_DBCONFIG_WRITABLE_SCHEMA]] <dt>SQLITE_DBCONFIG_WRITABLE_SCHEMA</dt> +** <dd>The SQLITE_DBCONFIG_WRITABLE_SCHEMA option activates or deactivates the +** "writable_schema" flag. This has the same effect and is logically equivalent +** to setting [PRAGMA writable_schema=ON] or [PRAGMA writable_schema=OFF]. +** The first argument to this setting is an integer which is 0 to disable +** the writable_schema, positive to enable writable_schema, or negative to +** leave the setting unchanged. The second parameter is a pointer to an +** integer into which is written 0 or 1 to indicate whether the writable_schema +** is enabled or disabled following this call. +** </dd> +** +** [[SQLITE_DBCONFIG_LEGACY_ALTER_TABLE]] +** <dt>SQLITE_DBCONFIG_LEGACY_ALTER_TABLE</dt> +** <dd>The SQLITE_DBCONFIG_LEGACY_ALTER_TABLE option activates or deactivates +** the legacy behavior of the [ALTER TABLE RENAME] command such it +** behaves as it did prior to [version 3.24.0] (2018-06-04). See the +** "Compatibility Notice" on the [ALTER TABLE RENAME documentation] for +** additional information. This feature can also be turned on and off +** using the [PRAGMA legacy_alter_table] statement. +** </dd> +** +** [[SQLITE_DBCONFIG_DQS_DML]] +** <dt>SQLITE_DBCONFIG_DQS_DML</td> +** <dd>The SQLITE_DBCONFIG_DQS_DML option activates or deactivates +** the legacy [double-quoted string literal] misfeature for DML statement +** only, that is DELETE, INSERT, SELECT, and UPDATE statements. The +** default value of this setting is determined by the [-DSQLITE_DQS] +** compile-time option. +** </dd> +** +** [[SQLITE_DBCONFIG_DQS_DDL]] +** <dt>SQLITE_DBCONFIG_DQS_DDL</td> +** <dd>The SQLITE_DBCONFIG_DQS option activates or deactivates +** the legacy [double-quoted string literal] misfeature for DDL statements, +** such as CREATE TABLE and CREATE INDEX. The +** default value of this setting is determined by the [-DSQLITE_DQS] +** compile-time option. ** </dd> ** </dl> */@@ -3204,7 +3304,12 @@ #define SQLITE_DBCONFIG_NO_CKPT_ON_CLOSE 1006 /* int int* */
#define SQLITE_DBCONFIG_ENABLE_QPSG 1007 /* int int* */ #define SQLITE_DBCONFIG_TRIGGER_EQP 1008 /* int int* */ #define SQLITE_DBCONFIG_RESET_DATABASE 1009 /* int int* */ -#define SQLITE_DBCONFIG_MAX 1009 /* Largest DBCONFIG */ +#define SQLITE_DBCONFIG_DEFENSIVE 1010 /* int int* */ +#define SQLITE_DBCONFIG_WRITABLE_SCHEMA 1011 /* int int* */ +#define SQLITE_DBCONFIG_LEGACY_ALTER_TABLE 1012 /* int int* */ +#define SQLITE_DBCONFIG_DQS_DML 1013 /* int int* */ +#define SQLITE_DBCONFIG_DQS_DDL 1014 /* int int* */ +#define SQLITE_DBCONFIG_MAX 1014 /* Largest DBCONFIG */ /* ** CAPI3REF: Enable Or Disable Extended Result Codes@@ -3361,7 +3466,7 @@ ** count, but those made as part of REPLACE constraint resolution are
** not. ^Changes to a view that are intercepted by INSTEAD OF triggers ** are not counted. ** -** This the [sqlite3_total_changes(D)] interface only reports the number +** The [sqlite3_total_changes(D)] interface only reports the number ** of rows that changed due to SQL statement run against database ** connection D. Any changes by other database connections are ignored. ** To detect changes against a database file from other database@@ -4005,9 +4110,9 @@ ** of how long that statement took to run. ^The profile callback
** time is in units of nanoseconds, however the current implementation ** is only capable of millisecond resolution so the six least significant ** digits in the time are meaningless. Future versions of SQLite -** might provide greater resolution on the profiler callback. The -** sqlite3_profile() function is considered experimental and is -** subject to change in future versions of SQLite. +** might provide greater resolution on the profiler callback. Invoking +** either [sqlite3_trace()] or [sqlite3_trace_v2()] will cancel the +** profile callback. */ SQLITE_API SQLITE_DEPRECATED void *sqlite3_trace(sqlite3*, void(*xTrace)(void*,const char*), void*);@@ -4421,6 +4526,8 @@ ** sqlite3_uri_boolean(F,P,B) returns B. If F is not a NULL pointer and
** is not a database file pathname pointer that SQLite passed into the xOpen ** VFS method, then the behavior of this routine is undefined and probably ** undesirable. +** +** See the [URI filename] documentation for additional information. */ SQLITE_API const char *sqlite3_uri_parameter(const char *zFilename, const char *zParam); SQLITE_API int sqlite3_uri_boolean(const char *zFile, const char *zParam, int bDefault);@@ -4642,9 +4749,24 @@ ** [sqlite3_finalize()] relatively soon. The current implementation acts
** on this hint by avoiding the use of [lookaside memory] so as not to ** deplete the limited store of lookaside memory. Future versions of ** SQLite may act on this hint differently. +** +** [[SQLITE_PREPARE_NORMALIZE]] <dt>SQLITE_PREPARE_NORMALIZE</dt> +** <dd>The SQLITE_PREPARE_NORMALIZE flag is a no-op. This flag used +** to be required for any prepared statement that wanted to use the +** [sqlite3_normalized_sql()] interface. However, the +** [sqlite3_normalized_sql()] interface is now available to all +** prepared statements, regardless of whether or not they use this +** flag. +** +** [[SQLITE_PREPARE_NO_VTAB]] <dt>SQLITE_PREPARE_NO_VTAB</dt> +** <dd>The SQLITE_PREPARE_NO_VTAB flag causes the SQL compiler +** to return an error (error code SQLITE_ERROR) if the statement uses +** any virtual tables. ** </dl> */ #define SQLITE_PREPARE_PERSISTENT 0x01 +#define SQLITE_PREPARE_NORMALIZE 0x02 +#define SQLITE_PREPARE_NO_VTAB 0x04 /* ** CAPI3REF: Compiling An SQL Statement@@ -4802,6 +4924,11 @@ ** [sqlite3_prepare16_v2()], or [sqlite3_prepare16_v3()].
** ^The sqlite3_expanded_sql(P) interface returns a pointer to a UTF-8 ** string containing the SQL text of prepared statement P with ** [bound parameters] expanded. +** ^The sqlite3_normalized_sql(P) interface returns a pointer to a UTF-8 +** string containing the normalized SQL text of prepared statement P. The +** semantics used to normalize a SQL statement are unspecified and subject +** to change. At a minimum, literal values will be replaced with suitable +** placeholders. ** ** ^(For example, if a prepared statement is created using the SQL ** text "SELECT $abc,:xyz" and if parameter $abc is bound to integer 2345@@ -4817,14 +4944,16 @@ ** ^The [SQLITE_TRACE_SIZE_LIMIT] compile-time option limits the size of
** bound parameter expansions. ^The [SQLITE_OMIT_TRACE] compile-time ** option causes sqlite3_expanded_sql() to always return NULL. ** -** ^The string returned by sqlite3_sql(P) is managed by SQLite and is -** automatically freed when the prepared statement is finalized. +** ^The strings returned by sqlite3_sql(P) and sqlite3_normalized_sql(P) +** are managed by SQLite and are automatically freed when the prepared +** statement is finalized. ** ^The string returned by sqlite3_expanded_sql(P), on the other hand, ** is obtained from [sqlite3_malloc()] and must be free by the application ** by passing it to [sqlite3_free()]. */ SQLITE_API const char *sqlite3_sql(sqlite3_stmt *pStmt); SQLITE_API char *sqlite3_expanded_sql(sqlite3_stmt *pStmt); +SQLITE_API const char *sqlite3_normalized_sql(sqlite3_stmt *pStmt); /* ** CAPI3REF: Determine If An SQL Statement Writes The Database@@ -4861,6 +4990,18 @@ ** [BEGIN|BEGIN EXCLUSIVE] commands do touch the database and so
** sqlite3_stmt_readonly() returns false for those commands. */ SQLITE_API int sqlite3_stmt_readonly(sqlite3_stmt *pStmt); + +/* +** CAPI3REF: Query The EXPLAIN Setting For A Prepared Statement +** METHOD: sqlite3_stmt +** +** ^The sqlite3_stmt_isexplain(S) interface returns 1 if the +** prepared statement S is an EXPLAIN statement, or 2 if the +** statement S is an EXPLAIN QUERY PLAN. +** ^The sqlite3_stmt_isexplain(S) interface returns 0 if S is +** an ordinary statement or a NULL pointer. +*/ +SQLITE_API int sqlite3_stmt_isexplain(sqlite3_stmt *pStmt); /* ** CAPI3REF: Determine If A Prepared Statement Has Been Reset@@ -5001,7 +5142,9 @@ **
** ^The fifth argument to the BLOB and string binding interfaces ** is a destructor used to dispose of the BLOB or ** string after SQLite has finished with it. ^The destructor is called -** to dispose of the BLOB or string even if the call to bind API fails. +** to dispose of the BLOB or string even if the call to the bind API fails, +** except the destructor is not called if the third parameter is a NULL +** pointer or the fourth parameter is negative. ** ^If the fifth argument is ** the special value [SQLITE_STATIC], then SQLite assumes that the ** information is in static, unmanaged space and does not need to be freed.@@ -5918,6 +6061,8 @@ ** <td>→ <td>Best numeric datatype of the value
** <tr><td><b>sqlite3_value_nochange </b> ** <td>→ <td>True if the column is unchanged in an UPDATE ** against a virtual table. +** <tr><td><b>sqlite3_value_frombind </b> +** <td>→ <td>True if value originated from a [bound parameter] ** </table></blockquote> ** ** <b>Details:</b>@@ -5979,6 +6124,11 @@ ** to be a NULL value. If sqlite3_value_nochange(X) is invoked anywhere other
** than within an [xUpdate] method call for an UPDATE statement, then ** the return value is arbitrary and meaningless. ** +** ^The sqlite3_value_frombind(X) interface returns non-zero if the +** value X originated from one of the [sqlite3_bind_int|sqlite3_bind()] +** interfaces. ^If X comes from an SQL literal value, or a table column, +** and expression, then sqlite3_value_frombind(X) returns zero. +** ** Please pay particular attention to the fact that the pointer returned ** from [sqlite3_value_blob()], [sqlite3_value_text()], or ** [sqlite3_value_text16()] can be invalidated by a subsequent call to@@ -6024,6 +6174,7 @@ SQLITE_API int sqlite3_value_bytes16(sqlite3_value*);
SQLITE_API int sqlite3_value_type(sqlite3_value*); SQLITE_API int sqlite3_value_numeric_type(sqlite3_value*); SQLITE_API int sqlite3_value_nochange(sqlite3_value*); +SQLITE_API int sqlite3_value_frombind(sqlite3_value*); /* ** CAPI3REF: Finding The Subtype Of SQL Values@@ -6759,7 +6910,7 @@ ** ^The sqlite3_db_filename(D,N) interface returns a pointer to a filename
** associated with database N of connection D. ^The main database file ** has the name "main". If there is no attached database N on the database ** connection D, or if database N is a temporary or in-memory database, then -** a NULL pointer is returned. +** this function will return either a NULL pointer or an empty string. ** ** ^The filename returned by this function is the output of the ** xFullPathname method of the [VFS]. ^In other words, the filename@@ -7314,6 +7465,9 @@ ** below are for version 2 and greater. */
int (*xSavepoint)(sqlite3_vtab *pVTab, int); int (*xRelease)(sqlite3_vtab *pVTab, int); int (*xRollbackTo)(sqlite3_vtab *pVTab, int); + /* The methods above are in versions 1 and 2 of the sqlite_module object. + ** Those below are for version 3 and greater. */ + int (*xShadowName)(const char*); }; /*@@ -8236,6 +8390,7 @@ #define SQLITE_TESTCTRL_RESERVE 14
#define SQLITE_TESTCTRL_OPTIMIZATIONS 15 #define SQLITE_TESTCTRL_ISKEYWORD 16 /* NOT USED */ #define SQLITE_TESTCTRL_SCRATCHMALLOC 17 /* NOT USED */ +#define SQLITE_TESTCTRL_INTERNAL_FUNCTIONS 17 #define SQLITE_TESTCTRL_LOCALTIME_FAULT 18 #define SQLITE_TESTCTRL_EXPLAIN_STMT 19 /* NOT USED */ #define SQLITE_TESTCTRL_ONCE_RESET_THRESHOLD 19@@ -8246,7 +8401,8 @@ #define SQLITE_TESTCTRL_ISINIT 23
#define SQLITE_TESTCTRL_SORTER_MMAP 24 #define SQLITE_TESTCTRL_IMPOSTER 25 #define SQLITE_TESTCTRL_PARSER_COVERAGE 26 -#define SQLITE_TESTCTRL_LAST 26 /* Largest TESTCTRL */ +#define SQLITE_TESTCTRL_RESULT_INTREAL 27 +#define SQLITE_TESTCTRL_LAST 27 /* Largest TESTCTRL */ /* ** CAPI3REF: SQL Keyword Checking@@ -9648,6 +9804,7 @@ ** [sqlite3_vtab_config()] interface that [virtual table] implementations
** can use to customize and optimize their behavior. ** ** <dl> +** [[SQLITE_VTAB_CONSTRAINT_SUPPORT]] ** <dt>SQLITE_VTAB_CONSTRAINT_SUPPORT ** <dd>Calls of the form ** [sqlite3_vtab_config](db,SQLITE_VTAB_CONSTRAINT_SUPPORT,X) are supported,@@ -10417,7 +10574,7 @@ int mxLevel; /* The largest iLevel value in the tree */
sqlite3_int64 iRowid; /* Rowid for current entry */ sqlite3_rtree_dbl rParentScore; /* Score of parent node */ int eParentWithin; /* Visibility of parent node */ - int eWithin; /* OUT: Visiblity */ + int eWithin; /* OUT: Visibility */ sqlite3_rtree_dbl rScore; /* OUT: Write the score here */ /* The following fields are only available in 3.8.11 and later */ sqlite3_value **apSqlParam; /* Original SQL values of parameters */@@ -10913,12 +11070,38 @@ ** this function, all changes that relate to a single table are visited
** consecutively. There is no chance that the iterator will visit a change ** the applies to table X, then one for table Y, and then later on visit ** another change for table X. +** +** The behavior of sqlite3changeset_start_v2() and its streaming equivalent +** may be modified by passing a combination of +** [SQLITE_CHANGESETSTART_INVERT | supported flags] as the 4th parameter. +** +** Note that the sqlite3changeset_start_v2() API is still <b>experimental</b> +** and therefore subject to change. */ SQLITE_API int sqlite3changeset_start( sqlite3_changeset_iter **pp, /* OUT: New changeset iterator handle */ int nChangeset, /* Size of changeset blob in bytes */ void *pChangeset /* Pointer to blob containing changeset */ ); +SQLITE_API int sqlite3changeset_start_v2( + sqlite3_changeset_iter **pp, /* OUT: New changeset iterator handle */ + int nChangeset, /* Size of changeset blob in bytes */ + void *pChangeset, /* Pointer to blob containing changeset */ + int flags /* SESSION_CHANGESETSTART_* flags */ +); + +/* +** CAPI3REF: Flags for sqlite3changeset_start_v2 +** +** The following flags may passed via the 4th parameter to +** [sqlite3changeset_start_v2] and [sqlite3changeset_start_v2_strm]: +** +** <dt>SQLITE_CHANGESETAPPLY_INVERT <dd> +** Invert the changeset while iterating through it. This is equivalent to +** inverting a changeset using sqlite3changeset_invert() before applying it. +** It is an error to specify this flag with a patchset. +*/ +#define SQLITE_CHANGESETSTART_INVERT 0x0002 /*@@ -10962,7 +11145,7 @@ ** affected by the current change. The buffer remains valid until either
** sqlite3changeset_next() is called on the iterator or until the ** conflict-handler function returns. If pnCol is not NULL, then *pnCol is ** set to the number of columns in the table affected by the change. If -** pbIncorrect is not NULL, then *pbIndirect is set to true (1) if the change +** pbIndirect is not NULL, then *pbIndirect is set to true (1) if the change ** is an indirect change, or false (0) otherwise. See the documentation for ** [sqlite3session_indirect()] for a description of direct and indirect ** changes. Finally, if pOp is not NULL, then *pOp is set to one of@@ -11573,7 +11756,7 @@ sqlite3_changeset_iter *p /* Handle describing change and conflict */
), void *pCtx, /* First argument passed to xConflict */ void **ppRebase, int *pnRebase, /* OUT: Rebase data */ - int flags /* Combination of SESSION_APPLY_* flags */ + int flags /* SESSION_CHANGESETAPPLY_* flags */ ); /*@@ -11591,8 +11774,14 @@ ** applied, or rolled back if an error occurs. Specifying this flag
** causes the sessions module to omit this savepoint. In this case, if the ** caller has an open transaction or savepoint when apply_v2() is called, ** it may revert the partially applied changeset by rolling it back. +** +** <dt>SQLITE_CHANGESETAPPLY_INVERT <dd> +** Invert the changeset before applying it. This is equivalent to inverting +** a changeset using sqlite3changeset_invert() before applying it. It is +** an error to specify this flag with a patchset. */ #define SQLITE_CHANGESETAPPLY_NOSAVEPOINT 0x0001 +#define SQLITE_CHANGESETAPPLY_INVERT 0x0002 /* ** CAPI3REF: Constants Passed To The Conflict Handler@@ -11823,7 +12012,7 @@ ** Argument pIn must point to a buffer containing a changeset nIn bytes
** in size. This function allocates and populates a buffer with a copy ** of the changeset rebased rebased according to the configuration of the ** rebaser object passed as the first argument. If successful, (*ppOut) -** is set to point to the new buffer containing the rebased changset and +** is set to point to the new buffer containing the rebased changeset and ** (*pnOut) to its size in bytes and SQLITE_OK returned. It is the ** responsibility of the caller to eventually free the new buffer using ** sqlite3_free(). Otherwise, if an error occurs, (*ppOut) and (*pnOut)@@ -11986,6 +12175,12 @@ sqlite3_changeset_iter **pp,
int (*xInput)(void *pIn, void *pData, int *pnData), void *pIn ); +SQLITE_API int sqlite3changeset_start_v2_strm( + sqlite3_changeset_iter **pp, + int (*xInput)(void *pIn, void *pData, int *pnData), + void *pIn, + int flags +); SQLITE_API int sqlite3session_changeset_strm( sqlite3_session *pSession, int (*xOutput)(void *pOut, const void *pData, int nData),@@ -12012,6 +12207,45 @@ int (*xOutput)(void *pOut, const void *pData, int nData),
void *pOut ); +/* +** CAPI3REF: Configure global parameters +** +** The sqlite3session_config() interface is used to make global configuration +** changes to the sessions module in order to tune it to the specific needs +** of the application. +** +** The sqlite3session_config() interface is not threadsafe. If it is invoked +** while any other thread is inside any other sessions method then the +** results are undefined. Furthermore, if it is invoked after any sessions +** related objects have been created, the results are also undefined. +** +** The first argument to the sqlite3session_config() function must be one +** of the SQLITE_SESSION_CONFIG_XXX constants defined below. The +** interpretation of the (void*) value passed as the second parameter and +** the effect of calling this function depends on the value of the first +** parameter. +** +** <dl> +** <dt>SQLITE_SESSION_CONFIG_STRMSIZE<dd> +** By default, the sessions module streaming interfaces attempt to input +** and output data in approximately 1 KiB chunks. This operand may be used +** to set and query the value of this configuration setting. The pointer +** passed as the second argument must point to a value of type (int). +** If this value is greater than 0, it is used as the new streaming data +** chunk size for both input and output. Before returning, the (int) value +** pointed to by pArg is set to the final value of the streaming interface +** chunk size. +** </dl> +** +** This function returns SQLITE_OK if successful, or an SQLite error code +** otherwise. +*/ +SQLITE_API int sqlite3session_config(int op, void *pArg); + +/* +** CAPI3REF: Values for sqlite3session_config(). +*/ +#define SQLITE_SESSION_CONFIG_STRMSIZE 1 /* ** Make sure we can call this stuff from C++.@@ -12145,12 +12379,8 @@ ** output by xInstCount().
** ** Usually, output parameter *piPhrase is set to the phrase number, *piCol ** to the column in which it occurs and *piOff the token offset of the -** first token of the phrase. The exception is if the table was created -** with the offsets=0 option specified. In this case *piOff is always -** set to -1. -** -** Returns SQLITE_OK if successful, or an error code (i.e. SQLITE_NOMEM) -** if an error occurs. +** first token of the phrase. Returns SQLITE_OK if successful, or an error +** code (i.e. SQLITE_NOMEM) if an error occurs. ** ** This API can be quite slow if used with an FTS5 table created with the ** "detail=none" or "detail=column" option.@@ -12191,7 +12421,7 @@ **
** Save the pointer passed as the second argument as the extension functions ** "auxiliary data". The pointer may then be retrieved by the current or any ** future invocation of the same fts5 extension function made as part of -** of the same MATCH query using the xGetAuxdata() API. +** the same MATCH query using the xGetAuxdata() API. ** ** Each extension function is allocated a single auxiliary data slot for ** each FTS query (MATCH expression). If the extension function is invoked@@ -12206,7 +12436,7 @@ **
** The xDelete callback, if one is specified, is also invoked on the ** auxiliary data pointer after the FTS5 query has finished. ** -** If an error (e.g. an OOM condition) occurs within this function, an +** If an error (e.g. an OOM condition) occurs within this function, ** the auxiliary data is set to NULL and an error code returned. If the ** xDelete parameter was not NULL, it is invoked on the auxiliary data ** pointer before returning.@@ -12439,11 +12669,11 @@ ** "first" and "place". If the user then queries for '1st + place',
** the tokenizer substitutes "first" for "1st" and the query works ** as expected. ** -** <li> By adding multiple synonyms for a single term to the FTS index. -** In this case, when tokenizing query text, the tokenizer may -** provide multiple synonyms for a single term within the document. -** FTS5 then queries the index for each synonym individually. For -** example, faced with the query: +** <li> By querying the index for all synonyms of each query term +** separately. In this case, when tokenizing query text, the +** tokenizer may provide multiple synonyms for a single term +** within the document. FTS5 then queries the index for each +** synonym individually. For example, faced with the query: ** ** <codeblock> ** ... MATCH 'first place'</codeblock>@@ -12467,7 +12697,7 @@ ** added to the FTS index for "i", "won", "first", "1st" and
** "place". ** ** This way, even if the tokenizer does not provide synonyms -** when tokenizing query text (it should not - to do would be +** when tokenizing query text (it should not - to do so would be ** inefficient), it doesn't matter if the user queries for ** 'first + place' or '1st + place', as there are entries in the ** FTS index corresponding to both forms of the first token.@@ -13232,7 +13462,7 @@ unsigned int htsize; /* Number of buckets in the hash table */
unsigned int count; /* Number of entries in this table */ HashElem *first; /* The first element of the array */ struct _ht { /* the hash table */ - int count; /* Number of entries with this hash */ + unsigned int count; /* Number of entries with this hash */ HashElem *chain; /* Pointer to first entry with this hash */ } *ht; };@@ -13373,99 +13603,94 @@ #define TK_PARTITION 84
#define TK_PRECEDING 85 #define TK_RANGE 86 #define TK_UNBOUNDED 87 -#define TK_REINDEX 88 -#define TK_RENAME 89 -#define TK_CTIME_KW 90 -#define TK_ANY 91 -#define TK_BITAND 92 -#define TK_BITOR 93 -#define TK_LSHIFT 94 -#define TK_RSHIFT 95 -#define TK_PLUS 96 -#define TK_MINUS 97 -#define TK_STAR 98 -#define TK_SLASH 99 -#define TK_REM 100 -#define TK_CONCAT 101 -#define TK_COLLATE 102 -#define TK_BITNOT 103 -#define TK_ON 104 -#define TK_INDEXED 105 -#define TK_STRING 106 -#define TK_JOIN_KW 107 -#define TK_CONSTRAINT 108 -#define TK_DEFAULT 109 -#define TK_NULL 110 -#define TK_PRIMARY 111 -#define TK_UNIQUE 112 -#define TK_CHECK 113 -#define TK_REFERENCES 114 -#define TK_AUTOINCR 115 -#define TK_INSERT 116 -#define TK_DELETE 117 -#define TK_UPDATE 118 -#define TK_SET 119 -#define TK_DEFERRABLE 120 -#define TK_FOREIGN 121 -#define TK_DROP 122 -#define TK_UNION 123 -#define TK_ALL 124 -#define TK_EXCEPT 125 -#define TK_INTERSECT 126 -#define TK_SELECT 127 -#define TK_VALUES 128 -#define TK_DISTINCT 129 -#define TK_DOT 130 -#define TK_FROM 131 -#define TK_JOIN 132 -#define TK_USING 133 -#define TK_ORDER 134 -#define TK_GROUP 135 -#define TK_HAVING 136 -#define TK_LIMIT 137 -#define TK_WHERE 138 -#define TK_INTO 139 -#define TK_NOTHING 140 -#define TK_FLOAT 141 -#define TK_BLOB 142 -#define TK_INTEGER 143 -#define TK_VARIABLE 144 -#define TK_CASE 145 -#define TK_WHEN 146 -#define TK_THEN 147 -#define TK_ELSE 148 -#define TK_INDEX 149 -#define TK_ALTER 150 -#define TK_ADD 151 -#define TK_WINDOW 152 -#define TK_OVER 153 -#define TK_FILTER 154 -#define TK_TRUEFALSE 155 -#define TK_ISNOT 156 -#define TK_FUNCTION 157 -#define TK_COLUMN 158 -#define TK_AGG_FUNCTION 159 -#define TK_AGG_COLUMN 160 -#define TK_UMINUS 161 -#define TK_UPLUS 162 -#define TK_TRUTH 163 -#define TK_REGISTER 164 -#define TK_VECTOR 165 -#define TK_SELECT_COLUMN 166 -#define TK_IF_NULL_ROW 167 -#define TK_ASTERISK 168 -#define TK_SPAN 169 -#define TK_END_OF_FILE 170 -#define TK_UNCLOSED_STRING 171 -#define TK_SPACE 172 -#define TK_ILLEGAL 173 - -/* The token codes above must all fit in 8 bits */ -#define TKFLG_MASK 0xff - -/* Flags that can be added to a token code when it is not -** being stored in a u8: */ -#define TKFLG_DONTFOLD 0x100 /* Omit constant folding optimizations */ +#define TK_EXCLUDE 88 +#define TK_GROUPS 89 +#define TK_OTHERS 90 +#define TK_TIES 91 +#define TK_REINDEX 92 +#define TK_RENAME 93 +#define TK_CTIME_KW 94 +#define TK_ANY 95 +#define TK_BITAND 96 +#define TK_BITOR 97 +#define TK_LSHIFT 98 +#define TK_RSHIFT 99 +#define TK_PLUS 100 +#define TK_MINUS 101 +#define TK_STAR 102 +#define TK_SLASH 103 +#define TK_REM 104 +#define TK_CONCAT 105 +#define TK_COLLATE 106 +#define TK_BITNOT 107 +#define TK_ON 108 +#define TK_INDEXED 109 +#define TK_STRING 110 +#define TK_JOIN_KW 111 +#define TK_CONSTRAINT 112 +#define TK_DEFAULT 113 +#define TK_NULL 114 +#define TK_PRIMARY 115 +#define TK_UNIQUE 116 +#define TK_CHECK 117 +#define TK_REFERENCES 118 +#define TK_AUTOINCR 119 +#define TK_INSERT 120 +#define TK_DELETE 121 +#define TK_UPDATE 122 +#define TK_SET 123 +#define TK_DEFERRABLE 124 +#define TK_FOREIGN 125 +#define TK_DROP 126 +#define TK_UNION 127 +#define TK_ALL 128 +#define TK_EXCEPT 129 +#define TK_INTERSECT 130 +#define TK_SELECT 131 +#define TK_VALUES 132 +#define TK_DISTINCT 133 +#define TK_DOT 134 +#define TK_FROM 135 +#define TK_JOIN 136 +#define TK_USING 137 +#define TK_ORDER 138 +#define TK_GROUP 139 +#define TK_HAVING 140 +#define TK_LIMIT 141 +#define TK_WHERE 142 +#define TK_INTO 143 +#define TK_NOTHING 144 +#define TK_FLOAT 145 +#define TK_BLOB 146 +#define TK_INTEGER 147 +#define TK_VARIABLE 148 +#define TK_CASE 149 +#define TK_WHEN 150 +#define TK_THEN 151 +#define TK_ELSE 152 +#define TK_INDEX 153 +#define TK_ALTER 154 +#define TK_ADD 155 +#define TK_WINDOW 156 +#define TK_OVER 157 +#define TK_FILTER 158 +#define TK_TRUEFALSE 159 +#define TK_ISNOT 160 +#define TK_FUNCTION 161 +#define TK_COLUMN 162 +#define TK_AGG_FUNCTION 163 +#define TK_AGG_COLUMN 164 +#define TK_UMINUS 165 +#define TK_UPLUS 166 +#define TK_TRUTH 167 +#define TK_REGISTER 168 +#define TK_VECTOR 169 +#define TK_SELECT_COLUMN 170 +#define TK_IF_NULL_ROW 171 +#define TK_ASTERISK 172 +#define TK_SPAN 173 +#define TK_SPACE 174 +#define TK_ILLEGAL 175 /************** End of parse.h ***********************************************/ /************** Continuing where we left off in sqliteInt.h ******************/@@ -13771,12 +13996,13 @@ ** -DSQLITE_BYTEORDER=0 is set, then byte-order is determined
** at run-time. */ #ifndef SQLITE_BYTEORDER -# if defined(i386) || defined(__i386__) || defined(_M_IX86) || \ - defined(__x86_64) || defined(__x86_64__) || defined(_M_X64) || \ - defined(_M_AMD64) || defined(_M_ARM) || defined(__x86) || \ - defined(__arm__) || defined(_M_ARM64) +# if defined(i386) || defined(__i386__) || defined(_M_IX86) || \ + defined(__x86_64) || defined(__x86_64__) || defined(_M_X64) || \ + defined(_M_AMD64) || defined(_M_ARM) || defined(__x86) || \ + defined(__ARMEL__) || defined(__AARCH64EL__) || defined(_M_ARM64) # define SQLITE_BYTEORDER 1234 -# elif defined(sparc) || defined(__ppc__) +# elif defined(sparc) || defined(__ppc__) || \ + defined(__ARMEB__) || defined(__AARCH64EB__) # define SQLITE_BYTEORDER 4321 # else # define SQLITE_BYTEORDER 0@@ -14397,9 +14623,6 @@
SQLITE_PRIVATE int sqlite3BtreeInsert(BtCursor*, const BtreePayload *pPayload, int flags, int seekResult); SQLITE_PRIVATE int sqlite3BtreeFirst(BtCursor*, int *pRes); -#ifndef SQLITE_OMIT_WINDOWFUNC -SQLITE_PRIVATE void sqlite3BtreeSkipNext(BtCursor*); -#endif SQLITE_PRIVATE int sqlite3BtreeLast(BtCursor*, int *pRes); SQLITE_PRIVATE int sqlite3BtreeNext(BtCursor*, int flags); SQLITE_PRIVATE int sqlite3BtreeEof(BtCursor*);@@ -14411,6 +14634,7 @@ #endif
SQLITE_PRIVATE int sqlite3BtreePayload(BtCursor*, u32 offset, u32 amt, void*); SQLITE_PRIVATE const void *sqlite3BtreePayloadFetch(BtCursor*, u32 *pAmt); SQLITE_PRIVATE u32 sqlite3BtreePayloadSize(BtCursor*); +SQLITE_PRIVATE sqlite3_int64 sqlite3BtreeMaxRecordSize(BtCursor*); SQLITE_PRIVATE char *sqlite3BtreeIntegrityCheck(Btree*, int *aRoot, int nRoot, int, int*); SQLITE_PRIVATE struct Pager *sqlite3BtreePager(Btree*);@@ -14650,12 +14874,11 @@ # endif
#endif /* -** The following macro converts a relative address in the p2 field -** of a VdbeOp structure into a negative number so that -** sqlite3VdbeAddOpList() knows that the address is relative. Calling -** the macro again restores the address. +** The following macro converts a label returned by sqlite3VdbeMakeLabel() +** into an index into the Parse.aLabel[] array that contains the resolved +** address of that label. */ -#define ADDR(X) (-1-(X)) +#define ADDR(X) (~(X)) /* ** The makefile scans the vdbe.c source file and creates the "opcodes.h"@@ -14757,25 +14980,25 @@ #define OP_IsTrue 88 /* synopsis: r[P2] = coalesce(r[P1]==TRUE,P3) ^ P4 */
#define OP_Offset 89 /* synopsis: r[P3] = sqlite_offset(P1) */ #define OP_Column 90 /* synopsis: r[P3]=PX */ #define OP_Affinity 91 /* synopsis: affinity(r[P1@P2]) */ -#define OP_BitAnd 92 /* same as TK_BITAND, synopsis: r[P3]=r[P1]&r[P2] */ -#define OP_BitOr 93 /* same as TK_BITOR, synopsis: r[P3]=r[P1]|r[P2] */ -#define OP_ShiftLeft 94 /* same as TK_LSHIFT, synopsis: r[P3]=r[P2]<<r[P1] */ -#define OP_ShiftRight 95 /* same as TK_RSHIFT, synopsis: r[P3]=r[P2]>>r[P1] */ -#define OP_Add 96 /* same as TK_PLUS, synopsis: r[P3]=r[P1]+r[P2] */ -#define OP_Subtract 97 /* same as TK_MINUS, synopsis: r[P3]=r[P2]-r[P1] */ -#define OP_Multiply 98 /* same as TK_STAR, synopsis: r[P3]=r[P1]*r[P2] */ -#define OP_Divide 99 /* same as TK_SLASH, synopsis: r[P3]=r[P2]/r[P1] */ -#define OP_Remainder 100 /* same as TK_REM, synopsis: r[P3]=r[P2]%r[P1] */ -#define OP_Concat 101 /* same as TK_CONCAT, synopsis: r[P3]=r[P2]+r[P1] */ -#define OP_MakeRecord 102 /* synopsis: r[P3]=mkrec(r[P1@P2]) */ -#define OP_BitNot 103 /* same as TK_BITNOT, synopsis: r[P2]= ~r[P1] */ -#define OP_Count 104 /* synopsis: r[P2]=count() */ -#define OP_ReadCookie 105 -#define OP_String8 106 /* same as TK_STRING, synopsis: r[P2]='P4' */ -#define OP_SetCookie 107 -#define OP_ReopenIdx 108 /* synopsis: root=P2 iDb=P3 */ -#define OP_OpenRead 109 /* synopsis: root=P2 iDb=P3 */ -#define OP_OpenWrite 110 /* synopsis: root=P2 iDb=P3 */ +#define OP_MakeRecord 92 /* synopsis: r[P3]=mkrec(r[P1@P2]) */ +#define OP_Count 93 /* synopsis: r[P2]=count() */ +#define OP_ReadCookie 94 +#define OP_SetCookie 95 +#define OP_BitAnd 96 /* same as TK_BITAND, synopsis: r[P3]=r[P1]&r[P2] */ +#define OP_BitOr 97 /* same as TK_BITOR, synopsis: r[P3]=r[P1]|r[P2] */ +#define OP_ShiftLeft 98 /* same as TK_LSHIFT, synopsis: r[P3]=r[P2]<<r[P1] */ +#define OP_ShiftRight 99 /* same as TK_RSHIFT, synopsis: r[P3]=r[P2]>>r[P1] */ +#define OP_Add 100 /* same as TK_PLUS, synopsis: r[P3]=r[P1]+r[P2] */ +#define OP_Subtract 101 /* same as TK_MINUS, synopsis: r[P3]=r[P2]-r[P1] */ +#define OP_Multiply 102 /* same as TK_STAR, synopsis: r[P3]=r[P1]*r[P2] */ +#define OP_Divide 103 /* same as TK_SLASH, synopsis: r[P3]=r[P2]/r[P1] */ +#define OP_Remainder 104 /* same as TK_REM, synopsis: r[P3]=r[P2]%r[P1] */ +#define OP_Concat 105 /* same as TK_CONCAT, synopsis: r[P3]=r[P2]+r[P1] */ +#define OP_ReopenIdx 106 /* synopsis: root=P2 iDb=P3 */ +#define OP_BitNot 107 /* same as TK_BITNOT, synopsis: r[P2]= ~r[P1] */ +#define OP_OpenRead 108 /* synopsis: root=P2 iDb=P3 */ +#define OP_OpenWrite 109 /* synopsis: root=P2 iDb=P3 */ +#define OP_String8 110 /* same as TK_STRING, synopsis: r[P2]='P4' */ #define OP_OpenDup 111 #define OP_OpenAutoindex 112 /* synopsis: nColumn=P2 */ #define OP_OpenEphemeral 113 /* synopsis: nColumn=P2 */@@ -14788,57 +15011,56 @@ #define OP_SeekHit 119 /* synopsis: seekHit=P2 */
#define OP_Sequence 120 /* synopsis: r[P2]=cursor[P1].ctr++ */ #define OP_NewRowid 121 /* synopsis: r[P2]=rowid */ #define OP_Insert 122 /* synopsis: intkey=r[P3] data=r[P2] */ -#define OP_InsertInt 123 /* synopsis: intkey=P3 data=r[P2] */ -#define OP_Delete 124 -#define OP_ResetCount 125 -#define OP_SorterCompare 126 /* synopsis: if key(P1)!=trim(r[P3],P4) goto P2 */ -#define OP_SorterData 127 /* synopsis: r[P2]=data */ -#define OP_RowData 128 /* synopsis: r[P2]=data */ -#define OP_Rowid 129 /* synopsis: r[P2]=rowid */ -#define OP_NullRow 130 -#define OP_SeekEnd 131 -#define OP_SorterInsert 132 /* synopsis: key=r[P2] */ -#define OP_IdxInsert 133 /* synopsis: key=r[P2] */ -#define OP_IdxDelete 134 /* synopsis: key=r[P2@P3] */ -#define OP_DeferredSeek 135 /* synopsis: Move P3 to P1.rowid if needed */ -#define OP_IdxRowid 136 /* synopsis: r[P2]=rowid */ -#define OP_Destroy 137 -#define OP_Clear 138 -#define OP_ResetSorter 139 -#define OP_CreateBtree 140 /* synopsis: r[P2]=root iDb=P1 flags=P3 */ -#define OP_Real 141 /* same as TK_FLOAT, synopsis: r[P2]=P4 */ -#define OP_SqlExec 142 -#define OP_ParseSchema 143 -#define OP_LoadAnalysis 144 -#define OP_DropTable 145 -#define OP_DropIndex 146 -#define OP_DropTrigger 147 -#define OP_IntegrityCk 148 -#define OP_RowSetAdd 149 /* synopsis: rowset(P1)=r[P2] */ -#define OP_Param 150 -#define OP_FkCounter 151 /* synopsis: fkctr[P1]+=P2 */ -#define OP_MemMax 152 /* synopsis: r[P1]=max(r[P1],r[P2]) */ -#define OP_OffsetLimit 153 /* synopsis: if r[P1]>0 then r[P2]=r[P1]+max(0,r[P3]) else r[P2]=(-1) */ -#define OP_AggInverse 154 /* synopsis: accum=r[P3] inverse(r[P2@P5]) */ -#define OP_AggStep 155 /* synopsis: accum=r[P3] step(r[P2@P5]) */ -#define OP_AggStep1 156 /* synopsis: accum=r[P3] step(r[P2@P5]) */ -#define OP_AggValue 157 /* synopsis: r[P3]=value N=P2 */ -#define OP_AggFinal 158 /* synopsis: accum=r[P1] N=P2 */ -#define OP_Expire 159 -#define OP_TableLock 160 /* synopsis: iDb=P1 root=P2 write=P3 */ -#define OP_VBegin 161 -#define OP_VCreate 162 -#define OP_VDestroy 163 -#define OP_VOpen 164 -#define OP_VColumn 165 /* synopsis: r[P3]=vcolumn(P2) */ -#define OP_VRename 166 -#define OP_Pagecount 167 -#define OP_MaxPgcnt 168 -#define OP_Trace 169 -#define OP_CursorHint 170 -#define OP_Noop 171 -#define OP_Explain 172 -#define OP_Abortable 173 +#define OP_Delete 123 +#define OP_ResetCount 124 +#define OP_SorterCompare 125 /* synopsis: if key(P1)!=trim(r[P3],P4) goto P2 */ +#define OP_SorterData 126 /* synopsis: r[P2]=data */ +#define OP_RowData 127 /* synopsis: r[P2]=data */ +#define OP_Rowid 128 /* synopsis: r[P2]=rowid */ +#define OP_NullRow 129 +#define OP_SeekEnd 130 +#define OP_SorterInsert 131 /* synopsis: key=r[P2] */ +#define OP_IdxInsert 132 /* synopsis: key=r[P2] */ +#define OP_IdxDelete 133 /* synopsis: key=r[P2@P3] */ +#define OP_DeferredSeek 134 /* synopsis: Move P3 to P1.rowid if needed */ +#define OP_IdxRowid 135 /* synopsis: r[P2]=rowid */ +#define OP_Destroy 136 +#define OP_Clear 137 +#define OP_ResetSorter 138 +#define OP_CreateBtree 139 /* synopsis: r[P2]=root iDb=P1 flags=P3 */ +#define OP_SqlExec 140 +#define OP_ParseSchema 141 +#define OP_LoadAnalysis 142 +#define OP_DropTable 143 +#define OP_DropIndex 144 +#define OP_Real 145 /* same as TK_FLOAT, synopsis: r[P2]=P4 */ +#define OP_DropTrigger 146 +#define OP_IntegrityCk 147 +#define OP_RowSetAdd 148 /* synopsis: rowset(P1)=r[P2] */ +#define OP_Param 149 +#define OP_FkCounter 150 /* synopsis: fkctr[P1]+=P2 */ +#define OP_MemMax 151 /* synopsis: r[P1]=max(r[P1],r[P2]) */ +#define OP_OffsetLimit 152 /* synopsis: if r[P1]>0 then r[P2]=r[P1]+max(0,r[P3]) else r[P2]=(-1) */ +#define OP_AggInverse 153 /* synopsis: accum=r[P3] inverse(r[P2@P5]) */ +#define OP_AggStep 154 /* synopsis: accum=r[P3] step(r[P2@P5]) */ +#define OP_AggStep1 155 /* synopsis: accum=r[P3] step(r[P2@P5]) */ +#define OP_AggValue 156 /* synopsis: r[P3]=value N=P2 */ +#define OP_AggFinal 157 /* synopsis: accum=r[P1] N=P2 */ +#define OP_Expire 158 +#define OP_TableLock 159 /* synopsis: iDb=P1 root=P2 write=P3 */ +#define OP_VBegin 160 +#define OP_VCreate 161 +#define OP_VDestroy 162 +#define OP_VOpen 163 +#define OP_VColumn 164 /* synopsis: r[P3]=vcolumn(P2) */ +#define OP_VRename 165 +#define OP_Pagecount 166 +#define OP_MaxPgcnt 167 +#define OP_Trace 168 +#define OP_CursorHint 169 +#define OP_Noop 170 +#define OP_Explain 171 +#define OP_Abortable 172 /* Properties such as "out2" or "jump" that are specified in ** comments following the "case" for each opcode in the vdbe.c@@ -14862,17 +15084,17 @@ /* 56 */ 0x0b, 0x0b, 0x01, 0x01, 0x01, 0x01, 0x00, 0x00,\
/* 64 */ 0x00, 0x00, 0x02, 0x02, 0x08, 0x00, 0x10, 0x10,\ /* 72 */ 0x10, 0x10, 0x00, 0x10, 0x10, 0x00, 0x00, 0x10,\ /* 80 */ 0x10, 0x00, 0x00, 0x02, 0x02, 0x02, 0x00, 0x00,\ -/* 88 */ 0x12, 0x20, 0x00, 0x00, 0x26, 0x26, 0x26, 0x26,\ -/* 96 */ 0x26, 0x26, 0x26, 0x26, 0x26, 0x26, 0x00, 0x12,\ -/* 104 */ 0x10, 0x10, 0x10, 0x00, 0x00, 0x00, 0x00, 0x00,\ +/* 88 */ 0x12, 0x20, 0x00, 0x00, 0x00, 0x10, 0x10, 0x00,\ +/* 96 */ 0x26, 0x26, 0x26, 0x26, 0x26, 0x26, 0x26, 0x26,\ +/* 104 */ 0x26, 0x26, 0x00, 0x12, 0x00, 0x00, 0x10, 0x00,\ /* 112 */ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,\ /* 120 */ 0x10, 0x10, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,\ -/* 128 */ 0x00, 0x10, 0x00, 0x00, 0x04, 0x04, 0x00, 0x00,\ -/* 136 */ 0x10, 0x10, 0x00, 0x00, 0x10, 0x10, 0x00, 0x00,\ -/* 144 */ 0x00, 0x00, 0x00, 0x00, 0x00, 0x06, 0x10, 0x00,\ -/* 152 */ 0x04, 0x1a, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,\ -/* 160 */ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x10,\ -/* 168 */ 0x10, 0x00, 0x00, 0x00, 0x00, 0x00,} +/* 128 */ 0x10, 0x00, 0x00, 0x04, 0x04, 0x00, 0x00, 0x10,\ +/* 136 */ 0x10, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, 0x00,\ +/* 144 */ 0x00, 0x10, 0x00, 0x00, 0x06, 0x10, 0x00, 0x04,\ +/* 152 */ 0x1a, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,\ +/* 160 */ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x10, 0x10,\ +/* 168 */ 0x00, 0x00, 0x00, 0x00, 0x00,} /* The sqlite3P2Values() routine is able to run faster if it knows ** the value of the largest JUMP opcode. The smaller the maximum@@ -14931,6 +15153,12 @@ #else
# define ExplainQueryPlan(P) # define ExplainQueryPlanPop(P) # define ExplainQueryPlanParent(P) 0 +# define sqlite3ExplainBreakpoint(A,B) /*no-op*/ +#endif +#if defined(SQLITE_DEBUG) && !defined(SQLITE_OMIT_EXPLAIN) +SQLITE_PRIVATE void sqlite3ExplainBreakpoint(const char*,const char*); +#else +# define sqlite3ExplainBreakpoint(A,B) /*no-op*/ #endif SQLITE_PRIVATE void sqlite3VdbeAddParseSchemaOp(Vdbe*,int,char*); SQLITE_PRIVATE void sqlite3VdbeChangeOpcode(Vdbe*, u32 addr, u8);@@ -14946,7 +15174,7 @@ SQLITE_PRIVATE void sqlite3VdbeAppendP4(Vdbe*, void *pP4, int p4type);
SQLITE_PRIVATE void sqlite3VdbeSetP4KeyInfo(Parse*, Index*); SQLITE_PRIVATE void sqlite3VdbeUsesBtree(Vdbe*, int); SQLITE_PRIVATE VdbeOp *sqlite3VdbeGetOp(Vdbe*, int); -SQLITE_PRIVATE int sqlite3VdbeMakeLabel(Vdbe*); +SQLITE_PRIVATE int sqlite3VdbeMakeLabel(Parse*); SQLITE_PRIVATE void sqlite3VdbeRunOnlyOnce(Vdbe*); SQLITE_PRIVATE void sqlite3VdbeReusable(Vdbe*); SQLITE_PRIVATE void sqlite3VdbeDelete(Vdbe*);@@ -14967,6 +15195,10 @@ SQLITE_PRIVATE void sqlite3VdbeCountChanges(Vdbe*);
SQLITE_PRIVATE sqlite3 *sqlite3VdbeDb(Vdbe*); SQLITE_PRIVATE u8 sqlite3VdbePrepareFlags(Vdbe*); SQLITE_PRIVATE void sqlite3VdbeSetSql(Vdbe*, const char *z, int n, u8); +#ifdef SQLITE_ENABLE_NORMALIZE +SQLITE_PRIVATE void sqlite3VdbeAddDblquoteStr(sqlite3*,Vdbe*,const char*); +SQLITE_PRIVATE int sqlite3VdbeUsesDoubleQuotedString(Vdbe*,const char*); +#endif SQLITE_PRIVATE void sqlite3VdbeSwap(Vdbe*,Vdbe*); SQLITE_PRIVATE VdbeOp *sqlite3VdbeTakeOpArray(Vdbe*, int*, int*); SQLITE_PRIVATE sqlite3_value *sqlite3VdbeGetBoundValue(Vdbe*, int, u8);@@ -15277,9 +15509,6 @@ SQLITE_PRIVATE int sqlite3PagerWalSupported(Pager *pPager);
SQLITE_PRIVATE int sqlite3PagerWalCallback(Pager *pPager); SQLITE_PRIVATE int sqlite3PagerOpenWal(Pager *pPager, int *pisOpen); SQLITE_PRIVATE int sqlite3PagerCloseWal(Pager *pPager, sqlite3*); -# ifdef SQLITE_DIRECT_OVERFLOW_READ -SQLITE_PRIVATE int sqlite3PagerUseWal(Pager *pPager, Pgno); -# endif # ifdef SQLITE_ENABLE_SNAPSHOT SQLITE_PRIVATE int sqlite3PagerSnapshotGet(Pager *pPager, sqlite3_snapshot **ppSnapshot); SQLITE_PRIVATE int sqlite3PagerSnapshotOpen(Pager *pPager, sqlite3_snapshot *pSnapshot);@@ -15287,8 +15516,10 @@ SQLITE_PRIVATE int sqlite3PagerSnapshotRecover(Pager *pPager);
SQLITE_PRIVATE int sqlite3PagerSnapshotCheck(Pager *pPager, sqlite3_snapshot *pSnapshot); SQLITE_PRIVATE void sqlite3PagerSnapshotUnlock(Pager *pPager); # endif -#else -# define sqlite3PagerUseWal(x,y) 0 +#endif + +#ifdef SQLITE_DIRECT_OVERFLOW_READ +SQLITE_PRIVATE int sqlite3PagerDirectReadOk(Pager *pPager, Pgno pgno); #endif #ifdef SQLITE_ENABLE_ZIPVFS@@ -15532,6 +15763,10 @@ SQLITE_PRIVATE int sqlite3HeaderSizePcache1(void);
/* Number of dirty pages as a percentage of the configured cache size */ SQLITE_PRIVATE int sqlite3PCachePercentDirty(PCache*); + +#ifdef SQLITE_DIRECT_OVERFLOW_READ +SQLITE_PRIVATE int sqlite3PCacheIsDirty(PCache *pCache); +#endif #endif /* _PCACHE_H_ */@@ -16038,12 +16273,14 @@ ** A hash table for built-in function definitions. (Application-defined
** functions use a regular table table from hash.h.) ** ** Hash each FuncDef structure into one of the FuncDefHash.a[] slots. -** Collisions are on the FuncDef.u.pHash chain. +** Collisions are on the FuncDef.u.pHash chain. Use the SQLITE_FUNC_HASH() +** macro to compute a hash on the function name. */ #define SQLITE_FUNC_HASH_SZ 23 struct FuncDefHash { FuncDef *a[SQLITE_FUNC_HASH_SZ]; /* Hash table for functions */ }; +#define SQLITE_FUNC_HASH(C,L) (((C)+(L))%SQLITE_FUNC_HASH_SZ) #ifdef SQLITE_USER_AUTHENTICATION /*@@ -16087,10 +16324,13 @@ #ifndef SQLITE_OMIT_DEPRECATED
/* This is an extra SQLITE_TRACE macro that indicates "legacy" tracing ** in the style of sqlite3_trace() */ -#define SQLITE_TRACE_LEGACY 0x80 +#define SQLITE_TRACE_LEGACY 0x40 /* Use the legacy xTrace */ +#define SQLITE_TRACE_XPROFILE 0x80 /* Use the legacy xProfile */ #else -#define SQLITE_TRACE_LEGACY 0 +#define SQLITE_TRACE_LEGACY 0 +#define SQLITE_TRACE_XPROFILE 0 #endif /* SQLITE_OMIT_DEPRECATED */ +#define SQLITE_TRACE_NONLEGACY_MASK 0x0f /* Normal flags */ /*@@ -16104,7 +16344,7 @@ sqlite3_mutex *mutex; /* Connection mutex */
Db *aDb; /* All backends */ int nDb; /* Number of backends currently in use */ u32 mDbFlags; /* flags recording internal state */ - u32 flags; /* flags settable by pragmas. See below */ + u64 flags; /* flags settable by pragmas. See below */ i64 lastRowid; /* ROWID of most recent insert (see above) */ i64 szMmap; /* Default mmap_size setting */ u32 nSchemaLock; /* Do not reset the schema when non-zero */@@ -16149,14 +16389,17 @@ int nExtension; /* Number of loaded extensions */
void **aExtension; /* Array of shared library handles */ int (*xTrace)(u32,void*,void*,void*); /* Trace function */ void *pTraceArg; /* Argument to the trace function */ +#ifndef SQLITE_OMIT_DEPRECATED void (*xProfile)(void*,const char*,u64); /* Profiling function */ void *pProfileArg; /* Argument to profile function */ +#endif void *pCommitArg; /* Argument to xCommitCallback() */ int (*xCommitCallback)(void*); /* Invoked at every commit. */ void *pRollbackArg; /* Argument to xRollbackCallback() */ void (*xRollbackCallback)(void*); /* Invoked at every commit. */ void *pUpdateArg; void (*xUpdateCallback)(void*,int, const char*,const char*,sqlite_int64); + Parse *pParse; /* Current parse */ #ifdef SQLITE_ENABLE_PREUPDATE_HOOK void *pPreUpdateArg; /* First argument to xPreUpdateCallback */ void (*xPreUpdateCallback)( /* Registered using sqlite3_preupdate_hook() */@@ -16269,14 +16512,21 @@ #define SQLITE_Fts3Tokenizer 0x00400000 /* Enable fts3_tokenizer(2) */
#define SQLITE_EnableQPSG 0x00800000 /* Query Planner Stability Guarantee*/ #define SQLITE_TriggerEQP 0x01000000 /* Show trigger EXPLAIN QUERY PLAN */ #define SQLITE_ResetDatabase 0x02000000 /* Reset the database */ +#define SQLITE_LegacyAlter 0x04000000 /* Legacy ALTER TABLE behaviour */ +#define SQLITE_NoSchemaError 0x08000000 /* Do not report schema parse errors*/ +#define SQLITE_Defensive 0x10000000 /* Input SQL is likely hostile */ +#define SQLITE_DqsDDL 0x20000000 /* dbl-quoted strings allowed in DDL*/ +#define SQLITE_DqsDML 0x40000000 /* dbl-quoted strings allowed in DML*/ /* Flags used only if debugging */ +#define HI(X) ((u64)(X)<<32) #ifdef SQLITE_DEBUG -#define SQLITE_SqlTrace 0x08000000 /* Debug print SQL as it executes */ -#define SQLITE_VdbeListing 0x10000000 /* Debug listings of VDBE programs */ -#define SQLITE_VdbeTrace 0x20000000 /* True to trace VDBE execution */ -#define SQLITE_VdbeAddopTrace 0x40000000 /* Trace sqlite3VdbeAddOp() calls */ -#define SQLITE_VdbeEQP 0x80000000 /* Debug EXPLAIN QUERY PLAN */ +#define SQLITE_SqlTrace HI(0x0001) /* Debug print SQL as it executes */ +#define SQLITE_VdbeListing HI(0x0002) /* Debug listings of VDBE progs */ +#define SQLITE_VdbeTrace HI(0x0004) /* True to trace VDBE execution */ +#define SQLITE_VdbeAddopTrace HI(0x0008) /* Trace sqlite3VdbeAddOp() calls */ +#define SQLITE_VdbeEQP HI(0x0010) /* Debug EXPLAIN QUERY PLAN */ +#define SQLITE_ParserTrace HI(0x0020) /* PRAGMA parser_trace=ON */ #endif /*@@ -16285,7 +16535,8 @@ */
#define DBFLAG_SchemaChange 0x0001 /* Uncommitted Hash table changes */ #define DBFLAG_PreferBuiltin 0x0002 /* Preference to built-in funcs */ #define DBFLAG_Vacuum 0x0004 /* Currently in a VACUUM */ -#define DBFLAG_SchemaKnownOk 0x0008 /* Schema is known to be valid */ +#define DBFLAG_VacuumInto 0x0008 /* Currently running VACUUM INTO */ +#define DBFLAG_SchemaKnownOk 0x0010 /* Schema is known to be valid */ /* ** Bits of the sqlite3.dbOptFlags field that are used by the@@ -16293,7 +16544,7 @@ ** sqlite3_test_control(SQLITE_TESTCTRL_OPTIMIZATIONS,...) interface to
** selectively disable various optimizations. */ #define SQLITE_QueryFlattener 0x0001 /* Query flattening */ - /* 0x0002 available for reuse */ +#define SQLITE_WindowFunc 0x0002 /* Use xInverse for window functions */ #define SQLITE_GroupByOrder 0x0004 /* GROUPBY cover of ORDERBY */ #define SQLITE_FactorOutConst 0x0008 /* Constant factoring */ #define SQLITE_DistinctOpt 0x0010 /* DISTINCT using indexes */@@ -16410,8 +16661,8 @@ #define SQLITE_FUNC_SLOCHNG 0x2000 /* "Slow Change". Value constant during a
** single query - might change over time */ #define SQLITE_FUNC_AFFINITY 0x4000 /* Built-in affinity() function */ #define SQLITE_FUNC_OFFSET 0x8000 /* Built-in sqlite_offset() function */ -#define SQLITE_FUNC_WINDOW 0x10000 /* Built-in window-only function */ -#define SQLITE_FUNC_WINDOW_SIZE 0x20000 /* Requires partition size as arg. */ +#define SQLITE_FUNC_WINDOW 0x00010000 /* Built-in window-only function */ +#define SQLITE_FUNC_INTERNAL 0x00040000 /* For use by NestedParse() only */ /* ** The following three macros, FUNCTION(), LIKEFUNC() and AGGREGATE() are@@ -16487,10 +16738,13 @@ SQLITE_INT_TO_PTR(arg), 0, xStep,xFinal,xValue,0,#zName, {0}}
#define AGGREGATE2(zName, nArg, arg, nc, xStep, xFinal, extraFlags) \ {nArg, SQLITE_UTF8|(nc*SQLITE_FUNC_NEEDCOLL)|extraFlags, \ SQLITE_INT_TO_PTR(arg), 0, xStep,xFinal,xFinal,0,#zName, {0}} - #define WAGGREGATE(zName, nArg, arg, nc, xStep, xFinal, xValue, xInverse, f) \ {nArg, SQLITE_UTF8|(nc*SQLITE_FUNC_NEEDCOLL)|f, \ SQLITE_INT_TO_PTR(arg), 0, xStep,xFinal,xValue,xInverse,#zName, {0}} +#define INTERNAL_FUNCTION(zName, nArg, xFunc) \ + {nArg, SQLITE_FUNC_INTERNAL|SQLITE_UTF8|SQLITE_FUNC_CONSTANT, \ + 0, 0, xFunc, 0, 0, 0, #zName, {0} } + /* ** All current savepoints are stored in a linked list starting at@@ -16725,6 +16979,7 @@ #define TF_OOOHidden 0x0080 /* Out-of-Order hidden columns */
#define TF_StatsUsed 0x0100 /* Query planner decisions affected by ** Index.aiRowLogEst[] values */ #define TF_HasNotNull 0x0200 /* Contains NOT NULL constraints */ +#define TF_Shadow 0x0400 /* True for a shadow table */ /* ** Test to see whether or not a table is a virtual table. This is@@ -16960,7 +17215,7 @@ LogEst szIdxRow; /* Estimated average row size in bytes */
u16 nKeyCol; /* Number of columns forming the key */ u16 nColumn; /* Number of columns stored in the index */ u8 onError; /* OE_Abort, OE_Ignore, OE_Replace, or OE_None */ - unsigned idxType:2; /* 1==UNIQUE, 2==PRIMARY KEY, 0==CREATE INDEX */ + unsigned idxType:2; /* 0:Normal 1:UNIQUE, 2:PRIMARY KEY, 3:IPK */ unsigned bUnordered:1; /* Use this index for == or IN queries only */ unsigned uniqNotNull:1; /* True if UNIQUE and NOT NULL for all columns */ unsigned isResized:1; /* True if resizeIndexObject() has been called */@@ -16968,6 +17223,7 @@ unsigned isCovering:1; /* True if this is a covering index */
unsigned noSkipScan:1; /* Do not try to use skip-scan if true */ unsigned hasStat1:1; /* aiRowLogEst values come from sqlite_stat1 */ unsigned bNoQuery:1; /* Do not use this index to optimize queries */ + unsigned bAscKeyBug:1; /* True if the bba7b69f9849b5bf bug applies */ #ifdef SQLITE_ENABLE_STAT3_OR_STAT4 int nSample; /* Number of elements in aSample[] */ int nSampleCol; /* Size of IndexSample.anEq[] and so on */@@ -16985,6 +17241,7 @@ */
#define SQLITE_IDXTYPE_APPDEF 0 /* Created using CREATE INDEX */ #define SQLITE_IDXTYPE_UNIQUE 1 /* Implements a UNIQUE constraint */ #define SQLITE_IDXTYPE_PRIMARYKEY 2 /* Is the PRIMARY KEY for the table */ +#define SQLITE_IDXTYPE_IPK 3 /* INTEGER PRIMARY KEY index */ /* Return true if index X is a PRIMARY KEY index */ #define IsPrimaryKeyIndex(X) ((X)->idxType==SQLITE_IDXTYPE_PRIMARYKEY)@@ -17010,6 +17267,12 @@ tRowcnt *anEq; /* Est. number of rows where the key equals this sample */
tRowcnt *anLt; /* Est. number of rows where key is less than this sample */ tRowcnt *anDLt; /* Est. number of distinct keys less than this sample */ }; + +/* +** Possible values to use within the flags argument to sqlite3GetToken(). +*/ +#define SQLITE_TOKEN_QUOTED 0x1 /* Token is a quoted identifier. */ +#define SQLITE_TOKEN_KEYWORD 0x2 /* Token is a keyword. */ /* ** Each token coming out of the lexer is an instance of@@ -17188,25 +17451,33 @@ ** TK_VARIABLE: variable number (always >= 1).
** TK_SELECT_COLUMN: column of the result vector */ i16 iAgg; /* Which entry in pAggInfo->aCol[] or ->aFunc[] */ i16 iRightJoinTable; /* If EP_FromJoin, the right table of the join */ - u8 op2; /* TK_REGISTER: original value of Expr.op + u8 op2; /* TK_REGISTER/TK_TRUTH: original value of Expr.op ** TK_COLUMN: the value of p5 for OP_Column ** TK_AGG_FUNCTION: nesting depth */ AggInfo *pAggInfo; /* Used by TK_AGG_COLUMN and TK_AGG_FUNCTION */ - Table *pTab; /* Table for TK_COLUMN expressions. Can be NULL - ** for a column of an index on an expression */ -#ifndef SQLITE_OMIT_WINDOWFUNC - Window *pWin; /* Window definition for window functions */ -#endif + union { + Table *pTab; /* TK_COLUMN: Table containing column. Can be NULL + ** for a column of an index on an expression */ + Window *pWin; /* TK_FUNCTION: Window definition for the func */ + struct { /* TK_IN, TK_SELECT, and TK_EXISTS */ + int iAddr; /* Subroutine entry address */ + int regReturn; /* Register used to hold return address */ + } sub; + } y; }; /* ** The following are the meanings of bits in the Expr.flags field. +** Value restrictions: +** +** EP_Agg == NC_HasAgg == SF_HasAgg +** EP_Win == NC_HasWin */ #define EP_FromJoin 0x000001 /* Originates in ON/USING clause of outer join */ -#define EP_Agg 0x000002 /* Contains one or more aggregate functions */ +#define EP_Distinct 0x000002 /* Aggregate function with DISTINCT keyword */ #define EP_HasFunc 0x000004 /* Contains one or more functions of any kind */ #define EP_FixedCol 0x000008 /* TK_Column with a known fixed value */ -#define EP_Distinct 0x000010 /* Aggregate function with DISTINCT keyword */ +#define EP_Agg 0x000010 /* Contains one or more aggregate functions */ #define EP_VarSelect 0x000020 /* pSelect is correlated, not constant */ #define EP_DblQuoted 0x000040 /* token.z was originally in "..." */ #define EP_InfixFunc 0x000080 /* True for an infix function: LIKE, GLOB, etc */@@ -17214,10 +17485,10 @@ #define EP_Collate 0x000100 /* Tree contains a TK_COLLATE operator */
#define EP_Generic 0x000200 /* Ignore COLLATE or affinity on this tree */ #define EP_IntValue 0x000400 /* Integer value contained in u.iValue */ #define EP_xIsSelect 0x000800 /* x.pSelect is valid (otherwise x.pList is) */ -#define EP_Skip 0x001000 /* COLLATE, AS, or UNLIKELY */ +#define EP_Skip 0x001000 /* Operator does not contribute to affinity */ #define EP_Reduced 0x002000 /* Expr struct EXPR_REDUCEDSIZE bytes only */ #define EP_TokenOnly 0x004000 /* Expr struct EXPR_TOKENONLYSIZE bytes only */ -#define EP_Static 0x008000 /* Held in memory not obtained from malloc() */ +#define EP_Win 0x008000 /* Contains window functions */ #define EP_MemToken 0x010000 /* Need to sqlite3DbFree() Expr.zToken */ #define EP_NoReduce 0x020000 /* Cannot EXPRDUP_REDUCE this Expr */ #define EP_Unlikely 0x040000 /* unlikely() or likelihood() function */@@ -17226,6 +17497,12 @@ #define EP_CanBeNull 0x100000 /* Can be null despite NOT NULL constraint */
#define EP_Subquery 0x200000 /* Tree contains a TK_SELECT operator */ #define EP_Alias 0x400000 /* Is an alias for a result set column */ #define EP_Leaf 0x800000 /* Expr.pLeft, .pRight, .u.pSelect all NULL */ +#define EP_WinFunc 0x1000000 /* TK_FUNCTION with Expr.y.pWin set */ +#define EP_Subrtn 0x2000000 /* Uses Expr.y.sub. TK_IN, _SELECT, or _EXISTS */ +#define EP_Quoted 0x4000000 /* TK_ID was originally quoted */ +#define EP_Static 0x8000000 /* Held in memory not obtained from malloc() */ +#define EP_IsTrue 0x10000000 /* Always has boolean value of TRUE */ +#define EP_IsFalse 0x20000000 /* Always has boolean value of FALSE */ /* ** The EP_Propagate mask is a set of properties that automatically propagate@@ -17241,6 +17518,8 @@ #define ExprHasProperty(E,P) (((E)->flags&(P))!=0)
#define ExprHasAllProperty(E,P) (((E)->flags&(P))==(P)) #define ExprSetProperty(E,P) (E)->flags|=(P) #define ExprClearProperty(E,P) (E)->flags&=~(P) +#define ExprAlwaysTrue(E) (((E)->flags&(EP_FromJoin|EP_IsTrue))==EP_IsTrue) +#define ExprAlwaysFalse(E) (((E)->flags&(EP_FromJoin|EP_IsFalse))==EP_IsFalse) /* The ExprSetVVAProperty() macro is used for Verification, Validation, ** and Accreditation only. It works like ExprSetProperty() during VVA@@ -17457,7 +17736,7 @@ } uNC;
NameContext *pNext; /* Next outer name context. NULL for outermost */ int nRef; /* Number of names resolved by this context */ int nErr; /* Number of errors encountered while resolving names */ - u16 ncFlags; /* Zero or more NC_* flags defined below */ + int ncFlags; /* Zero or more NC_* flags defined below */ Select *pWinSelect; /* SELECT statement for any window functions */ };@@ -17465,8 +17744,9 @@ /*
** Allowed values for the NameContext, ncFlags field. ** ** Value constraints (all checked via assert()): -** NC_HasAgg == SF_HasAgg +** NC_HasAgg == SF_HasAgg == EP_Agg ** NC_MinMaxAgg == SF_MinMaxAgg == SQLITE_FUNC_MINMAX +** NC_HasWin == EP_Win ** */ #define NC_AllowAgg 0x0001 /* Aggregate functions are allowed here */@@ -17482,6 +17762,8 @@ #define NC_UUpsert 0x0200 /* True if uNC.pUpsert is used */
#define NC_MinMaxAgg 0x1000 /* min/max aggregates seen. See note above */ #define NC_Complex 0x2000 /* True if a function or subquery seen */ #define NC_AllowWin 0x4000 /* Window functions are allowed here */ +#define NC_HasWin 0x8000 /* One or more window functions seen */ +#define NC_IsDDL 0x10000 /* Resolving names in a CREATE statement */ /* ** An instance of the following object describes a single ON CONFLICT@@ -17769,16 +18051,17 @@ u8 mayAbort; /* True if statement may throw an ABORT exception */
u8 hasCompound; /* Need to invoke convertCompoundSelectToSubquery() */ u8 okConstFactor; /* OK to factor out constants */ u8 disableLookaside; /* Number of times lookaside has been disabled */ + u8 disableVtab; /* Disable all virtual tables for this parse */ int nRangeReg; /* Size of the temporary register block */ int iRangeReg; /* First register in temporary register block */ int nErr; /* Number of errors seen */ int nTab; /* Number of previously allocated VDBE cursors */ int nMem; /* Number of memory cells used so far */ - int nOpAlloc; /* Number of slots allocated for Vdbe.aOp[] */ int szOpAlloc; /* Bytes of memory space allocated for Vdbe.aOp[] */ int iSelfTab; /* Table associated with an index on expr, or negative ** of the base register during check-constraint eval */ - int nLabel; /* Number of labels used */ + int nLabel; /* The *negative* of the number of labels used */ + int nLabelAlloc; /* Number of slots in aLabel */ int *aLabel; /* Space to hold the labels */ ExprList *pConstExpr;/* Constant expressions */ Token constraintName;/* Name of the constraint currently being parsed */@@ -17795,6 +18078,7 @@ #endif
AutoincInfo *pAinc; /* Information about AUTOINCREMENT counters */ Parse *pToplevel; /* Parse structure for main program (or NULL) */ Table *pTriggerTab; /* Table triggers are being coded for */ + Parse *pParentParse; /* Parent parser if this parser is nested */ int addrCrTab; /* Address of OP_CreateBtree opcode on CREATE TABLE */ u32 nQueryLoop; /* Est number of iterations of a query (10*log2(N)) */ u32 oldmask; /* Mask of old.* columns referenced */@@ -17838,7 +18122,9 @@ VList *pVList; /* Mapping between variable names and numbers */
Vdbe *pReprepare; /* VM being reprepared (sqlite3Reprepare()) */ const char *zTail; /* All SQL text past the last semicolon parsed */ Table *pNewTable; /* A table being constructed by CREATE TABLE */ - Index *pNewIndex; /* An index being constructed by CREATE INDEX */ + Index *pNewIndex; /* An index being constructed by CREATE INDEX. + ** Also used to hold redundant UNIQUE constraints + ** during a RENAME COLUMN */ Trigger *pNewTrigger; /* Trigger under construct by a CREATE TRIGGER */ const char *zAuthContext; /* The 6th parameter to db->xAuth callbacks */ #ifndef SQLITE_OMIT_VIRTUALTABLE@@ -17911,6 +18197,7 @@ ** OPFLAG_AUXDELETE == BTREE_AUXDELETE
*/ #define OPFLAG_NCHANGE 0x01 /* OP_Insert: Set to update db->nChange */ /* Also used in P2 (not P5) of OP_Delete */ +#define OPFLAG_NOCHNG 0x01 /* OP_VColumn nochange for UPDATE */ #define OPFLAG_EPHEM 0x01 /* OP_Column: Ephemeral output is ok */ #define OPFLAG_LASTROWID 0x20 /* Set to update db->lastRowid */ #define OPFLAG_ISUPDATE 0x04 /* This OP_Insert is an sql UPDATE */@@ -18065,6 +18352,7 @@ char **pzErrMsg; /* Error message stored here */
int iDb; /* 0 for main database. 1 for TEMP, 2.. for ATTACHed */ int rc; /* Result code stored here */ u32 mInitFlags; /* Flags controlling error messages */ + u32 nInitRow; /* Number of rows processed */ } InitData; /*@@ -18125,10 +18413,14 @@ */
void (*xVdbeBranch)(void*,unsigned iSrcLine,u8 eThis,u8 eMx); /* Callback */ void *pVdbeBranchArg; /* 1st argument */ #endif +#ifdef SQLITE_ENABLE_DESERIALIZE + sqlite3_int64 mxMemdbSize; /* Default max memdb size */ +#endif #ifndef SQLITE_UNTESTABLE int (*xTestCallback)(int); /* Invoked by sqlite3FaultSim() */ #endif int bLocaltimeFault; /* True to fail localtime() calls */ + int bInternalFunctions; /* Internal SQL functions are visible */ int iOnceResetThreshold; /* When to reset OP_Once counters */ u32 szSorterRef; /* Min size in bytes to use sorter-refs */ };@@ -18227,7 +18519,7 @@ };
#endif /* SQLITE_DEBUG */ /* -** This object is used in varioius ways, all related to window functions +** This object is used in various ways, all related to window functions ** ** (1) A single instance of this structure is attached to the ** the Expr.pWin field for each window function in an expression tree.@@ -18242,15 +18534,18 @@ ** (3) The terms of the WINDOW clause of a SELECT are instances of this
** object on a linked list attached to Select.pWinDefn. ** ** The uses (1) and (2) are really the same Window object that just happens -** to be accessible in two different ways. Use (3) is are separate objects. +** to be accessible in two different ways. Use case (3) are separate objects. */ struct Window { char *zName; /* Name of window (may be NULL) */ + char *zBase; /* Name of base window for chaining (may be NULL) */ ExprList *pPartition; /* PARTITION BY clause */ ExprList *pOrderBy; /* ORDER BY clause */ - u8 eType; /* TK_RANGE or TK_ROWS */ + u8 eFrmType; /* TK_RANGE, TK_GROUPS, TK_ROWS, or 0 */ u8 eStart; /* UNBOUNDED, CURRENT, PRECEDING or FOLLOWING */ u8 eEnd; /* UNBOUNDED, CURRENT, PRECEDING or FOLLOWING */ + u8 bImplicitFrame; /* True if frame was implicitly specified */ + u8 eExclude; /* TK_NO, TK_CURRENT, TK_TIES, TK_GROUP, or 0 */ Expr *pStart; /* Expression for "<expr> PRECEDING" */ Expr *pEnd; /* Expression for "<expr> FOLLOWING" */ Window *pNextWin; /* Next window function belonging to this SELECT */@@ -18261,17 +18556,19 @@ int regAccum;
int regResult; int csrApp; /* Function cursor (used by min/max) */ int regApp; /* Function register (also used by min/max) */ - int regPart; /* First in a set of registers holding PARTITION BY - ** and ORDER BY values for the window */ + int regPart; /* Array of registers for PARTITION BY values */ Expr *pOwner; /* Expression object this window is attached to */ int nBufferCol; /* Number of columns in buffer table */ int iArgCol; /* Offset of first argument for this function */ + int regOne; /* Register containing constant value 1 */ + int regStartRowid; + int regEndRowid; }; #ifndef SQLITE_OMIT_WINDOWFUNC SQLITE_PRIVATE void sqlite3WindowDelete(sqlite3*, Window*); SQLITE_PRIVATE void sqlite3WindowListDelete(sqlite3 *db, Window *p); -SQLITE_PRIVATE Window *sqlite3WindowAlloc(Parse*, int, int, Expr*, int , Expr*); +SQLITE_PRIVATE Window *sqlite3WindowAlloc(Parse*, int, int, Expr*, int , Expr*, u8); SQLITE_PRIVATE void sqlite3WindowAttach(Parse*, Expr*, Window*); SQLITE_PRIVATE int sqlite3WindowCompare(Parse*, Window*, Window*); SQLITE_PRIVATE void sqlite3WindowCodeInit(Parse*, Window*);@@ -18282,6 +18579,8 @@ SQLITE_PRIVATE void sqlite3WindowUpdate(Parse*, Window*, Window*, FuncDef*);
SQLITE_PRIVATE Window *sqlite3WindowDup(sqlite3 *db, Expr *pOwner, Window *p); SQLITE_PRIVATE Window *sqlite3WindowListDup(sqlite3 *db, Window *p); SQLITE_PRIVATE void sqlite3WindowFunctions(void); +SQLITE_PRIVATE void sqlite3WindowChain(Parse*, Window*, Window*); +SQLITE_PRIVATE Window *sqlite3WindowAssemble(Parse*, Window*, ExprList*, ExprList*, Token*); #else # define sqlite3WindowDelete(a,b) # define sqlite3WindowFunctions()@@ -18382,6 +18681,7 @@ ** Internal function prototypes
*/ SQLITE_PRIVATE int sqlite3StrICmp(const char*,const char*); SQLITE_PRIVATE int sqlite3Strlen30(const char*); +#define sqlite3Strlen30NN(C) (strlen(C)&0x3fffffff) SQLITE_PRIVATE char *sqlite3ColumnType(Column*,char*); #define sqlite3StrNICmp sqlite3_strnicmp@@ -18470,8 +18770,12 @@ # define sqlite3MutexWarnOnContention(x)
#endif #ifndef SQLITE_OMIT_FLOATING_POINT +# define EXP754 (((u64)0x7ff)<<52) +# define MAN754 ((((u64)1)<<52)-1) +# define IsNaN(X) (((X)&EXP754)==EXP754 && ((X)&MAN754)!=0) SQLITE_PRIVATE int sqlite3IsNaN(double); #else +# define IsNaN(X) 0 # define sqlite3IsNaN(X) 0 #endif@@ -18498,6 +18802,7 @@ #if defined(SQLITE_DEBUG)
SQLITE_PRIVATE void sqlite3TreeViewExpr(TreeView*, const Expr*, u8); SQLITE_PRIVATE void sqlite3TreeViewBareExprList(TreeView*, const ExprList*, const char*); SQLITE_PRIVATE void sqlite3TreeViewExprList(TreeView*, const ExprList*, u8, const char*); +SQLITE_PRIVATE void sqlite3TreeViewSrcList(TreeView*, const SrcList*); SQLITE_PRIVATE void sqlite3TreeViewSelect(TreeView*, const Select*, u8); SQLITE_PRIVATE void sqlite3TreeViewWith(TreeView*, const With*, u8); #ifndef SQLITE_OMIT_WINDOWFUNC@@ -18509,7 +18814,9 @@
SQLITE_PRIVATE void sqlite3SetString(char **, sqlite3*, const char*); SQLITE_PRIVATE void sqlite3ErrorMsg(Parse*, const char*, ...); +SQLITE_PRIVATE int sqlite3ErrorToParser(sqlite3*,int); SQLITE_PRIVATE void sqlite3Dequote(char*); +SQLITE_PRIVATE void sqlite3DequoteExpr(Expr*); SQLITE_PRIVATE void sqlite3TokenInit(Token*,char*); SQLITE_PRIVATE int sqlite3KeywordCode(const unsigned char*, int); SQLITE_PRIVATE int sqlite3RunParser(Parse*, const char*, char **);@@ -18527,10 +18834,12 @@ SQLITE_PRIVATE Expr *sqlite3Expr(sqlite3*,int,const char*);
SQLITE_PRIVATE void sqlite3ExprAttachSubtrees(sqlite3*,Expr*,Expr*,Expr*); SQLITE_PRIVATE Expr *sqlite3PExpr(Parse*, int, Expr*, Expr*); SQLITE_PRIVATE void sqlite3PExprAddSelect(Parse*, Expr*, Select*); -SQLITE_PRIVATE Expr *sqlite3ExprAnd(sqlite3*,Expr*, Expr*); +SQLITE_PRIVATE Expr *sqlite3ExprAnd(Parse*,Expr*, Expr*); +SQLITE_PRIVATE Expr *sqlite3ExprSimplifiedAndOr(Expr*); SQLITE_PRIVATE Expr *sqlite3ExprFunction(Parse*,ExprList*, Token*, int); SQLITE_PRIVATE void sqlite3ExprAssignVarNumber(Parse*, Expr*, u32); SQLITE_PRIVATE void sqlite3ExprDelete(sqlite3*, Expr*); +SQLITE_PRIVATE void sqlite3ExprUnmapAndDelete(Parse*, Expr*); SQLITE_PRIVATE ExprList *sqlite3ExprListAppend(Parse*,ExprList*,Expr*); SQLITE_PRIVATE ExprList *sqlite3ExprListAppendVector(Parse*,ExprList*,IdList*,Expr*); SQLITE_PRIVATE void sqlite3ExprListSetSortOrder(ExprList*,int);@@ -18538,6 +18847,7 @@ SQLITE_PRIVATE void sqlite3ExprListSetName(Parse*,ExprList*,Token*,int);
SQLITE_PRIVATE void sqlite3ExprListSetSpan(Parse*,ExprList*,const char*,const char*); SQLITE_PRIVATE void sqlite3ExprListDelete(sqlite3*, ExprList*); SQLITE_PRIVATE u32 sqlite3ExprListFlags(const ExprList*); +SQLITE_PRIVATE int sqlite3IndexHasDuplicateRootPage(Index*); SQLITE_PRIVATE int sqlite3Init(sqlite3*, char**); SQLITE_PRIVATE int sqlite3InitCallback(void*, int, char**, char**); SQLITE_PRIVATE int sqlite3InitOne(sqlite3*, int, char**, u32);@@ -18571,6 +18881,11 @@ SQLITE_PRIVATE void sqlite3AddCollateType(Parse*, Token*);
SQLITE_PRIVATE void sqlite3EndTable(Parse*,Token*,Token*,u8,Select*); SQLITE_PRIVATE int sqlite3ParseUri(const char*,const char*,unsigned int*, sqlite3_vfs**,char**,char **); +#ifdef SQLITE_HAS_CODEC +SQLITE_PRIVATE int sqlite3CodecQueryParameters(sqlite3*,const char*,const char*); +#else +# define sqlite3CodecQueryParameters(A,B,C) 0 +#endif SQLITE_PRIVATE Btree *sqlite3DbNameToBtree(sqlite3*,const char*); #ifdef SQLITE_UNTESTABLE@@ -18623,8 +18938,8 @@ SQLITE_PRIVATE void sqlite3Insert(Parse*, SrcList*, Select*, IdList*, int, Upsert*);
SQLITE_PRIVATE void *sqlite3ArrayAllocate(sqlite3*,void*,int,int*,int*); SQLITE_PRIVATE IdList *sqlite3IdListAppend(Parse*, IdList*, Token*); SQLITE_PRIVATE int sqlite3IdListIndex(IdList*,const char*); -SQLITE_PRIVATE SrcList *sqlite3SrcListEnlarge(sqlite3*, SrcList*, int, int); -SQLITE_PRIVATE SrcList *sqlite3SrcListAppend(sqlite3*, SrcList*, Token*, Token*); +SQLITE_PRIVATE SrcList *sqlite3SrcListEnlarge(Parse*, SrcList*, int, int); +SQLITE_PRIVATE SrcList *sqlite3SrcListAppend(Parse*, SrcList*, Token*, Token*); SQLITE_PRIVATE SrcList *sqlite3SrcListAppendFromTerm(Parse*, SrcList*, Token*, Token*, Token*, Select*, Expr*, IdList*); SQLITE_PRIVATE void sqlite3SrcListIndexedBy(Parse *, SrcList *, Token *);@@ -18691,8 +19006,8 @@ SQLITE_PRIVATE Table *sqlite3LocateTableItem(Parse*,u32 flags,struct SrcList_item *);
SQLITE_PRIVATE Index *sqlite3FindIndex(sqlite3*,const char*, const char*); SQLITE_PRIVATE void sqlite3UnlinkAndDeleteTable(sqlite3*,int,const char*); SQLITE_PRIVATE void sqlite3UnlinkAndDeleteIndex(sqlite3*,int,const char*); -SQLITE_PRIVATE void sqlite3Vacuum(Parse*,Token*); -SQLITE_PRIVATE int sqlite3RunVacuum(char**, sqlite3*, int); +SQLITE_PRIVATE void sqlite3Vacuum(Parse*,Token*,Expr*); +SQLITE_PRIVATE int sqlite3RunVacuum(char**, sqlite3*, int, sqlite3_value*); SQLITE_PRIVATE char *sqlite3NameFromToken(sqlite3*, Token*); SQLITE_PRIVATE int sqlite3ExprCompare(Parse*,Expr*, Expr*, int); SQLITE_PRIVATE int sqlite3ExprCompareSkip(Expr*, Expr*, int);@@ -18735,6 +19050,7 @@ Parse*,Table*,Trigger*,int,int,int,i16,u8,u8,u8,int);
SQLITE_PRIVATE void sqlite3GenerateRowIndexDelete(Parse*, Table*, int, int, int*, int); SQLITE_PRIVATE int sqlite3GenerateIndexKey(Parse*, Index*, int, int, int, int*,Index*,int); SQLITE_PRIVATE void sqlite3ResolvePartIdxLabel(Parse*,int); +SQLITE_PRIVATE int sqlite3ExprReferencesUpdatedColumn(Expr*,int*,int); SQLITE_PRIVATE void sqlite3GenerateConstraintChecks(Parse*,Table*,int*,int,int,int,int, u8,u8,int,int*,int*,Upsert*); #ifdef SQLITE_ENABLE_NULL_TRIM@@ -18755,6 +19071,7 @@ SQLITE_PRIVATE ExprList *sqlite3ExprListDup(sqlite3*,ExprList*,int);
SQLITE_PRIVATE SrcList *sqlite3SrcListDup(sqlite3*,SrcList*,int); SQLITE_PRIVATE IdList *sqlite3IdListDup(sqlite3*,IdList*); SQLITE_PRIVATE Select *sqlite3SelectDup(sqlite3*,Select*,int); +SQLITE_PRIVATE FuncDef *sqlite3FunctionSearch(int,const char*); SQLITE_PRIVATE void sqlite3InsertBuiltinFuncs(FuncDef*,int); SQLITE_PRIVATE FuncDef *sqlite3FindFunction(sqlite3*,const char*,int,u8,u8); SQLITE_PRIVATE void sqlite3RegisterBuiltinFunctions(void);@@ -18831,6 +19148,7 @@ SQLITE_PRIVATE int sqlite3FixSelect(DbFixer*, Select*);
SQLITE_PRIVATE int sqlite3FixExpr(DbFixer*, Expr*); SQLITE_PRIVATE int sqlite3FixExprList(DbFixer*, ExprList*); SQLITE_PRIVATE int sqlite3FixTriggerStep(DbFixer*, TriggerStep*); +SQLITE_PRIVATE int sqlite3RealSameAsInt(double,sqlite3_int64); SQLITE_PRIVATE int sqlite3AtoF(const char *z, double*, int, u8); SQLITE_PRIVATE int sqlite3GetInt32(const char *, int*); SQLITE_PRIVATE int sqlite3Atoi(const char*);@@ -18912,6 +19230,7 @@ SQLITE_PRIVATE Expr *sqlite3ExprAddCollateToken(Parse *pParse, Expr*, const Token*, int);
SQLITE_PRIVATE Expr *sqlite3ExprAddCollateString(Parse*,Expr*,const char*); SQLITE_PRIVATE Expr *sqlite3ExprSkipCollate(Expr*); SQLITE_PRIVATE int sqlite3CheckCollSeq(Parse *, CollSeq *); +SQLITE_PRIVATE int sqlite3WritableSchema(sqlite3*); SQLITE_PRIVATE int sqlite3CheckObjectName(Parse *, const char *); SQLITE_PRIVATE void sqlite3VdbeSetChanges(sqlite3 *, int); SQLITE_PRIVATE int sqlite3AddInt64(i64*,i64);@@ -18931,6 +19250,9 @@ SQLITE_PRIVATE void sqlite3ValueSetStr(sqlite3_value*, int, const void *,u8,
void(*)(void*)); SQLITE_PRIVATE void sqlite3ValueSetNull(sqlite3_value*); SQLITE_PRIVATE void sqlite3ValueFree(sqlite3_value*); +#ifndef SQLITE_UNTESTABLE +SQLITE_PRIVATE void sqlite3ResultIntReal(sqlite3_context*); +#endif SQLITE_PRIVATE sqlite3_value *sqlite3ValueNew(sqlite3 *); #ifndef SQLITE_OMIT_UTF16 SQLITE_PRIVATE char *sqlite3Utf16to8(sqlite3 *, const void*, int, u8);@@ -18960,14 +19282,15 @@ SQLITE_PRIVATE void sqlite3AlterRenameColumn(Parse*, SrcList*, Token*, Token*);
SQLITE_PRIVATE int sqlite3GetToken(const unsigned char *, int *); SQLITE_PRIVATE void sqlite3NestedParse(Parse*, const char*, ...); SQLITE_PRIVATE void sqlite3ExpirePreparedStatements(sqlite3*, int); -SQLITE_PRIVATE int sqlite3CodeSubselect(Parse*, Expr *, int, int); +SQLITE_PRIVATE void sqlite3CodeRhsOfIN(Parse*, Expr*, int); +SQLITE_PRIVATE int sqlite3CodeSubselect(Parse*, Expr*); SQLITE_PRIVATE void sqlite3SelectPrep(Parse*, Select*, NameContext*); SQLITE_PRIVATE void sqlite3SelectWrongNumTermsError(Parse *pParse, Select *p); SQLITE_PRIVATE int sqlite3MatchSpanName(const char*, const char*, const char*, const char*); SQLITE_PRIVATE int sqlite3ResolveExprNames(NameContext*, Expr*); SQLITE_PRIVATE int sqlite3ResolveExprListNames(NameContext*, ExprList*); SQLITE_PRIVATE void sqlite3ResolveSelectNames(Parse*, Select*, NameContext*); -SQLITE_PRIVATE void sqlite3ResolveSelfReference(Parse*,Table*,int,Expr*,ExprList*); +SQLITE_PRIVATE int sqlite3ResolveSelfReference(Parse*,Table*,int,Expr*,ExprList*); SQLITE_PRIVATE int sqlite3ResolveOrderGroupBy(Parse*, Select*, ExprList*, const char*); SQLITE_PRIVATE void sqlite3ColumnDefault(Vdbe *, Table *, int, int); SQLITE_PRIVATE void sqlite3AlterFinishAddColumn(Parse *, Token *);@@ -19115,6 +19438,9 @@ SQLITE_PRIVATE sqlite3_int64 sqlite3StmtCurrentTime(sqlite3_context*);
SQLITE_PRIVATE int sqlite3VdbeParameterIndex(Vdbe*, const char*, int); SQLITE_PRIVATE int sqlite3TransferBindings(sqlite3_stmt *, sqlite3_stmt *); SQLITE_PRIVATE void sqlite3ParserReset(Parse*); +#ifdef SQLITE_ENABLE_NORMALIZE +SQLITE_PRIVATE char *sqlite3Normalize(Vdbe*, const char*); +#endif SQLITE_PRIVATE int sqlite3Reprepare(Vdbe*); SQLITE_PRIVATE void sqlite3ExprListCheckLength(Parse*, ExprList*, const char*); SQLITE_PRIVATE CollSeq *sqlite3BinaryCompareCollSeq(Parse *, Expr *, Expr *);@@ -19209,7 +19535,7 @@ */
#define IN_INDEX_NOOP_OK 0x0001 /* OK to return IN_INDEX_NOOP */ #define IN_INDEX_MEMBERSHIP 0x0002 /* IN operator used for membership test */ #define IN_INDEX_LOOP 0x0004 /* IN operator used as a loop */ -SQLITE_PRIVATE int sqlite3FindInIndex(Parse *, Expr *, u32, int*, int*); +SQLITE_PRIVATE int sqlite3FindInIndex(Parse *, Expr *, u32, int*, int*, int*); SQLITE_PRIVATE int sqlite3JournalOpen(sqlite3_vfs *, const char *, sqlite3_file *, int, int); SQLITE_PRIVATE int sqlite3JournalSize(sqlite3_vfs *);@@ -19489,8 +19815,15 @@ /* EVIDENCE-OF: R-38720-18127 The default setting is determined by the
** SQLITE_ALLOW_COVERING_INDEX_SCAN compile-time option, or is "on" if ** that compile-time option is omitted. */ -#ifndef SQLITE_ALLOW_COVERING_INDEX_SCAN +#if !defined(SQLITE_ALLOW_COVERING_INDEX_SCAN) # define SQLITE_ALLOW_COVERING_INDEX_SCAN 1 +#else +# if !SQLITE_ALLOW_COVERING_INDEX_SCAN +# error "Compile-time disabling of covering index scan using the\ + -DSQLITE_ALLOW_COVERING_INDEX_SCAN=0 option is deprecated.\ + Contact SQLite developers if this is a problem for you, and\ + delete this #error macro to continue with your build." +# endif #endif /* The minimum PMA size is set to this value multiplied by the database@@ -19525,6 +19858,13 @@ # define SQLITE_DEFAULT_LOOKASIDE 1200,100
#endif +/* The default maximum size of an in-memory database created using +** sqlite3_deserialize() +*/ +#ifndef SQLITE_MEMDB_DEFAULT_MAXSIZE +# define SQLITE_MEMDB_DEFAULT_MAXSIZE 1073741824 +#endif + /* ** The following singleton contains the global configuration for ** the SQLite library.@@ -19572,12 +19912,16 @@ #ifdef SQLITE_VDBE_COVERAGE
0, /* xVdbeBranch */ 0, /* pVbeBranchArg */ #endif +#ifdef SQLITE_ENABLE_DESERIALIZE + SQLITE_MEMDB_DEFAULT_MAXSIZE, /* mxMemdbSize */ +#endif #ifndef SQLITE_UNTESTABLE 0, /* xTestCallback */ #endif 0, /* bLocaltimeFault */ + 0, /* bInternalFunctions */ 0x7ffffffe, /* iOnceResetThreshold */ - SQLITE_DEFAULT_SORTERREF_SIZE /* szSorterRef */ + SQLITE_DEFAULT_SORTERREF_SIZE, /* szSorterRef */ }; /*@@ -19906,12 +20250,12 @@ #define MEM_Str 0x0002 /* Value is a string */
#define MEM_Int 0x0004 /* Value is an integer */ #define MEM_Real 0x0008 /* Value is a real number */ #define MEM_Blob 0x0010 /* Value is a BLOB */ -#define MEM_AffMask 0x001f /* Mask of affinity bits */ -/* Available 0x0020 */ -/* Available 0x0040 */ +#define MEM_IntReal 0x0020 /* MEM_Int that stringifies like MEM_Real */ +#define MEM_AffMask 0x003f /* Mask of affinity bits */ +#define MEM_FromBind 0x0040 /* Value originates from sqlite3_bind() */ #define MEM_Undefined 0x0080 /* Value is undefined */ #define MEM_Cleared 0x0100 /* NULL set by OP_Null, not from data */ -#define MEM_TypeMask 0xc1ff /* Mask of type bits */ +#define MEM_TypeMask 0xc1bf /* Mask of type bits */ /* Whenever Mem contains a valid string or blob representation, one of@@ -19944,6 +20288,12 @@ #define MemSetTypeFlag(p, f) \
((p)->flags = ((p)->flags&~(MEM_TypeMask|MEM_Zero))|f) /* +** True if Mem X is a NULL-nochng type. +*/ +#define MemNullNochng(X) \ + ((X)->flags==(MEM_Null|MEM_Zero) && (X)->n==0 && (X)->u.nZero==0) + +/* ** Return true if a memory cell is not marked as invalid. This macro ** is for use inside assert() statements only. */@@ -19996,6 +20346,9 @@ ** N is the number of bits.
*/ typedef unsigned bft; /* Bit Field Type */ +/* The ScanStatus object holds a single value for the +** sqlite3_stmt_scanstatus() interface. +*/ typedef struct ScanStatus ScanStatus; struct ScanStatus { int addrExplain; /* OP_Explain for loop */@@ -20006,6 +20359,19 @@ LogEst nEst; /* Estimated output rows per loop */
char *zName; /* Name of table or index */ }; +/* The DblquoteStr object holds the text of a double-quoted +** string for a prepared statement. A linked list of these objects +** is constructed during statement parsing and is held on Vdbe.pDblStr. +** When computing a normalized SQL statement for an SQL statement, that +** list is consulted for each double-quoted identifier to see if the +** identifier should really be a string literal. +*/ +typedef struct DblquoteStr DblquoteStr; +struct DblquoteStr { + DblquoteStr *pNextStr; /* Next string literal in the list */ + char z[8]; /* Dequoted value for the string */ +}; + /* ** An instance of the virtual machine. This structure contains the complete ** state of the virtual machine.@@ -20025,28 +20391,29 @@ u32 cacheCtr; /* VdbeCursor row cache generation counter */
int pc; /* The program counter */ int rc; /* Value to return */ int nChange; /* Number of db changes made since last reset */ - int iStatement; /* Statement number (or 0 if has not opened stmt) */ + int iStatement; /* Statement number (or 0 if has no opened stmt) */ i64 iCurrentTime; /* Value of julianday('now') for this statement */ i64 nFkConstraint; /* Number of imm. FK constraints this VM */ i64 nStmtDefCons; /* Number of def. constraints when stmt started */ i64 nStmtDefImmCons; /* Number of def. imm constraints when stmt started */ + Mem *aMem; /* The memory locations */ + Mem **apArg; /* Arguments to currently executing user function */ + VdbeCursor **apCsr; /* One element of this array for each open cursor */ + Mem *aVar; /* Values for the OP_Variable opcode. */ /* When allocating a new Vdbe object, all of the fields below should be ** initialized to zero or NULL */ Op *aOp; /* Space to hold the virtual machine's program */ - Mem *aMem; /* The memory locations */ - Mem **apArg; /* Arguments to currently executing user function */ + int nOp; /* Number of instructions in the program */ + int nOpAlloc; /* Slots allocated for aOp[] */ Mem *aColName; /* Column names to return */ Mem *pResultSet; /* Pointer to an array of results */ char *zErrMsg; /* Error message written here */ - VdbeCursor **apCsr; /* One element of this array for each open cursor */ - Mem *aVar; /* Values for the OP_Variable opcode. */ VList *pVList; /* Name of variables */ #ifndef SQLITE_OMIT_TRACE i64 startTime; /* Time when query started - used for profiling */ #endif - int nOp; /* Number of instructions in the program */ #ifdef SQLITE_DEBUG int rcApp; /* errcode set by sqlite3_result_error_code() */ u32 nWrite; /* Number of write operations that have occurred */@@ -20067,6 +20434,10 @@ yDbMask btreeMask; /* Bitmask of db->aDb[] entries referenced */
yDbMask lockMask; /* Subset of btreeMask that requires a lock */ u32 aCounter[7]; /* Counters used by sqlite3_stmt_status() */ char *zSql; /* Text of the SQL statement that generated this */ +#ifdef SQLITE_ENABLE_NORMALIZE + char *zNormSql; /* Normalization of the associated SQL statement */ + DblquoteStr *pDblStr; /* List of double-quoted string literals */ +#endif void *pFree; /* Free this when deleting the vdbe */ VdbeFrame *pFrame; /* Parent frame */ VdbeFrame *pDelFrame; /* List of frame objects to free on VM reset */@@ -20129,7 +20500,9 @@ int sqlite2BtreeKeyCompare(BtCursor *, const void *, int, int, int *);
SQLITE_PRIVATE int sqlite3VdbeIdxKeyCompare(sqlite3*,VdbeCursor*,UnpackedRecord*,int*); SQLITE_PRIVATE int sqlite3VdbeIdxRowid(sqlite3*, BtCursor*, i64*); SQLITE_PRIVATE int sqlite3VdbeExec(Vdbe*); +#ifndef SQLITE_OMIT_EXPLAIN SQLITE_PRIVATE int sqlite3VdbeList(Vdbe*); +#endif SQLITE_PRIVATE int sqlite3VdbeHalt(Vdbe*); SQLITE_PRIVATE int sqlite3VdbeChangeEncoding(Mem *, int); SQLITE_PRIVATE int sqlite3VdbeMemTooBig(Mem*);@@ -20168,7 +20541,9 @@ SQLITE_PRIVATE int sqlite3VdbeMemFinalize(Mem*, FuncDef*);
#ifndef SQLITE_OMIT_WINDOWFUNC SQLITE_PRIVATE int sqlite3VdbeMemAggValue(Mem*, Mem*, FuncDef*); #endif +#ifndef SQLITE_OMIT_EXPLAIN SQLITE_PRIVATE const char *sqlite3OpcodeName(int); +#endif SQLITE_PRIVATE int sqlite3VdbeMemGrow(Mem *pMem, int n, int preserve); SQLITE_PRIVATE int sqlite3VdbeMemClearAndResize(Mem *pMem, int n); SQLITE_PRIVATE int sqlite3VdbeCloseStatement(Vdbe *, int);@@ -21000,7 +21375,7 @@ }else if( parseHhMmSs(zDate, p)==0 ){
return 0; }else if( sqlite3StrICmp(zDate,"now")==0 && sqlite3NotPureFunc(context) ){ return setDateTimeToCurrent(context, p); - }else if( sqlite3AtoF(zDate, &r, sqlite3Strlen30(zDate), SQLITE_UTF8) ){ + }else if( sqlite3AtoF(zDate, &r, sqlite3Strlen30(zDate), SQLITE_UTF8)>0 ){ setRawDateNumber(p, r); return 0; }@@ -21334,7 +21709,7 @@ ** weekday N where 0==Sunday, 1==Monday, and so forth. If the
** date is already on the appropriate weekday, this is a no-op. */ if( sqlite3_strnicmp(z, "weekday ", 8)==0 - && sqlite3AtoF(&z[8], &r, sqlite3Strlen30(&z[8]), SQLITE_UTF8) + && sqlite3AtoF(&z[8], &r, sqlite3Strlen30(&z[8]), SQLITE_UTF8)>0 && (n=(int)r)==r && n>=0 && r<7 ){ sqlite3_int64 Z; computeYMD_HMS(p);@@ -21393,7 +21768,7 @@ case '9': {
double rRounder; int i; for(n=1; z[n] && z[n]!=':' && !sqlite3Isspace(z[n]); n++){} - if( !sqlite3AtoF(z, &r, n, SQLITE_UTF8) ){ + if( sqlite3AtoF(z, &r, n, SQLITE_UTF8)<=0 ){ rc = 1; break; }@@ -26867,6 +27242,9 @@ if( db->nVdbeExec>0 ){
db->u1.isInterrupted = 1; } db->lookaside.bDisable++; + if( db->pParse ){ + db->pParse->rc = SQLITE_NOMEM_BKPT; + } } }@@ -27023,6 +27401,12 @@ { 'S', 0, 0, etSRCLIST, 0, 0 },
{ 'r', 10, 1, etORDINAL, 0, 0 }, }; +/* Floating point constants used for rounding */ +static const double arRound[] = { + 5.0e-01, 5.0e-02, 5.0e-03, 5.0e-04, 5.0e-05, + 5.0e-06, 5.0e-07, 5.0e-08, 5.0e-09, 5.0e-10, +}; + /* ** If SQLITE_OMIT_FLOATING_POINT is defined, then none of the floating point ** conversions will work.@@ -27060,7 +27444,8 @@ */
static void setStrAccumError(StrAccum *p, u8 eError){ assert( eError==SQLITE_NOMEM || eError==SQLITE_TOOBIG ); p->accError = eError; - p->nAlloc = 0; + if( p->mxAlloc ) sqlite3_str_reset(p); + if( eError==SQLITE_TOOBIG ) sqlite3ErrorToParser(p->db, eError); } /*@@ -27079,6 +27464,28 @@ if( p->nArg<=p->nUsed ) return 0;
return (char*)sqlite3_value_text(p->apArg[p->nUsed++]); } +/* +** Allocate memory for a temporary buffer needed for printf rendering. +** +** If the requested size of the temp buffer is larger than the size +** of the output buffer in pAccum, then cause an SQLITE_TOOBIG error. +** Do the size check before the memory allocation to prevent rogue +** SQL from requesting large allocations using the precision or width +** field of the printf() function. +*/ +static char *printfTempBuf(sqlite3_str *pAccum, sqlite3_int64 n){ + char *z; + if( pAccum->accError ) return 0; + if( n>pAccum->nAlloc && n>pAccum->mxAlloc ){ + setStrAccumError(pAccum, SQLITE_TOOBIG); + return 0; + } + z = sqlite3DbMallocRaw(pAccum->db, n); + if( z==0 ){ + setStrAccumError(pAccum, SQLITE_NOMEM); + } + return z; +} /* ** On machines with a small stack size, you can redefine the@@ -27161,6 +27568,9 @@ /* Find out what flags are present */
flag_leftjustify = flag_prefix = cThousand = flag_alternateform = flag_altform2 = flag_zeropad = 0; done = 0; + width = 0; + flag_long = 0; + precision = -1; do{ switch( c ){ case '-': flag_leftjustify = 1; break;@@ -27171,80 +27581,93 @@ case '!': flag_altform2 = 1; break;
case '0': flag_zeropad = 1; break; case ',': cThousand = ','; break; default: done = 1; break; - } - }while( !done && (c=(*++fmt))!=0 ); - /* Get the field width */ - if( c=='*' ){ - if( bArgList ){ - width = (int)getIntArg(pArgList); - }else{ - width = va_arg(ap,int); - } - if( width<0 ){ - flag_leftjustify = 1; - width = width >= -2147483647 ? -width : 0; - } - c = *++fmt; - }else{ - unsigned wx = 0; - while( c>='0' && c<='9' ){ - wx = wx*10 + c - '0'; - c = *++fmt; - } - testcase( wx>0x7fffffff ); - width = wx & 0x7fffffff; - } - assert( width>=0 ); + case 'l': { + flag_long = 1; + c = *++fmt; + if( c=='l' ){ + c = *++fmt; + flag_long = 2; + } + done = 1; + break; + } + case '1': case '2': case '3': case '4': case '5': + case '6': case '7': case '8': case '9': { + unsigned wx = c - '0'; + while( (c = *++fmt)>='0' && c<='9' ){ + wx = wx*10 + c - '0'; + } + testcase( wx>0x7fffffff ); + width = wx & 0x7fffffff; #ifdef SQLITE_PRINTF_PRECISION_LIMIT - if( width>SQLITE_PRINTF_PRECISION_LIMIT ){ - width = SQLITE_PRINTF_PRECISION_LIMIT; - } + if( width>SQLITE_PRINTF_PRECISION_LIMIT ){ + width = SQLITE_PRINTF_PRECISION_LIMIT; + } #endif - - /* Get the precision */ - if( c=='.' ){ - c = *++fmt; - if( c=='*' ){ - if( bArgList ){ - precision = (int)getIntArg(pArgList); - }else{ - precision = va_arg(ap,int); + if( c!='.' && c!='l' ){ + done = 1; + }else{ + fmt--; + } + break; } - c = *++fmt; - if( precision<0 ){ - precision = precision >= -2147483647 ? -precision : -1; + case '*': { + if( bArgList ){ + width = (int)getIntArg(pArgList); + }else{ + width = va_arg(ap,int); + } + if( width<0 ){ + flag_leftjustify = 1; + width = width >= -2147483647 ? -width : 0; + } +#ifdef SQLITE_PRINTF_PRECISION_LIMIT + if( width>SQLITE_PRINTF_PRECISION_LIMIT ){ + width = SQLITE_PRINTF_PRECISION_LIMIT; + } +#endif + if( (c = fmt[1])!='.' && c!='l' ){ + c = *++fmt; + done = 1; + } + break; } - }else{ - unsigned px = 0; - while( c>='0' && c<='9' ){ - px = px*10 + c - '0'; + case '.': { c = *++fmt; - } - testcase( px>0x7fffffff ); - precision = px & 0x7fffffff; - } - }else{ - precision = -1; - } - assert( precision>=(-1) ); + if( c=='*' ){ + if( bArgList ){ + precision = (int)getIntArg(pArgList); + }else{ + precision = va_arg(ap,int); + } + if( precision<0 ){ + precision = precision >= -2147483647 ? -precision : -1; + } + c = *++fmt; + }else{ + unsigned px = 0; + while( c>='0' && c<='9' ){ + px = px*10 + c - '0'; + c = *++fmt; + } + testcase( px>0x7fffffff ); + precision = px & 0x7fffffff; + } #ifdef SQLITE_PRINTF_PRECISION_LIMIT - if( precision>SQLITE_PRINTF_PRECISION_LIMIT ){ - precision = SQLITE_PRINTF_PRECISION_LIMIT; - } + if( precision>SQLITE_PRINTF_PRECISION_LIMIT ){ + precision = SQLITE_PRINTF_PRECISION_LIMIT; + } #endif - - - /* Get the conversion type modifier */ - if( c=='l' ){ - flag_long = 1; - c = *++fmt; - if( c=='l' ){ - flag_long = 2; - c = *++fmt; + if( c=='l' ){ + --fmt; + }else{ + done = 1; + } + break; + } } - }else{ - flag_long = 0; - } + }while( !done && (c=(*++fmt))!=0 ); + /* Fetch the info entry for the field */ infop = &fmtinfo[0]; xtype = etINVALID;@@ -27329,12 +27752,11 @@ if( precision<etBUFSIZE-10-etBUFSIZE/3 ){
nOut = etBUFSIZE; zOut = buf; }else{ - u64 n = (u64)precision + 10 + precision/3; - zOut = zExtra = sqlite3Malloc( n ); - if( zOut==0 ){ - setStrAccumError(pAccum, SQLITE_NOMEM); - return; - } + u64 n; + n = (u64)precision + 10; + if( cThousand ) n += precision/3; + zOut = zExtra = printfTempBuf(pAccum, n); + if( zOut==0 ) return; nOut = (int)n; } bufpt = &zOut[nOut-1];@@ -27403,8 +27825,18 @@ prefix = flag_prefix;
} if( xtype==etGENERIC && precision>0 ) precision--; testcase( precision>0xfff ); - for(idx=precision&0xfff, rounder=0.5; idx>0; idx--, rounder*=0.1){} - if( xtype==etFLOAT ) realvalue += rounder; + idx = precision & 0xfff; + rounder = arRound[idx%10]; + while( idx>=10 ){ rounder *= 1.0e-10; idx -= 10; } + if( xtype==etFLOAT ){ + double rx = (double)realvalue; + sqlite3_uint64 u; + int ex; + memcpy(&u, &rx, sizeof(u)); + ex = -1023 + (int)((u>>52)&0x7ff); + if( precision+(ex/3) < 15 ) rounder += realvalue*3e-16; + realvalue += rounder; + } /* Normalize realvalue to within 10.0 > realvalue >= 1.0 */ exp = 0; if( sqlite3IsNaN((double)realvalue) ){@@ -27453,12 +27885,12 @@ e2 = 0;
}else{ e2 = exp; } - if( MAX(e2,0)+(i64)precision+(i64)width > etBUFSIZE - 15 ){ - bufpt = zExtra - = sqlite3Malloc( MAX(e2,0)+(i64)precision+(i64)width+15 ); - if( bufpt==0 ){ - setStrAccumError(pAccum, SQLITE_NOMEM); - return; + { + i64 szBufNeeded; /* Size of a temporary buffer needed */ + szBufNeeded = MAX(e2,0)+(i64)precision+(i64)width+15; + if( szBufNeeded > etBUFSIZE ){ + bufpt = zExtra = printfTempBuf(pAccum, szBufNeeded); + if( bufpt==0 ) return; } } zOut = bufpt;@@ -27682,11 +28114,8 @@ }
needQuote = !isnull && xtype==etSQLESCAPE2; n += i + 3; if( n>etBUFSIZE ){ - bufpt = zExtra = sqlite3Malloc( n ); - if( bufpt==0 ){ - setStrAccumError(pAccum, SQLITE_NOMEM); - return; - } + bufpt = zExtra = printfTempBuf(pAccum, n); + if( bufpt==0 ) return; }else{ bufpt = buf; }@@ -27776,9 +28205,8 @@ testcase(p->accError==SQLITE_NOMEM);
return 0; } if( p->mxAlloc==0 ){ - N = p->nAlloc - p->nChar - 1; setStrAccumError(p, SQLITE_TOOBIG); - return N; + return p->nAlloc - p->nChar - 1; }else{ char *zOld = isMalloced(p) ? p->zText : 0; i64 szNew = p->nChar;@@ -27850,7 +28278,7 @@ SQLITE_API void sqlite3_str_append(sqlite3_str *p, const char *z, int N){
assert( z!=0 || N==0 ); assert( p->zText!=0 || p->nChar==0 || p->accError ); assert( N>=0 ); - assert( p->accError==0 || p->nAlloc==0 ); + assert( p->accError==0 || p->nAlloc==0 || p->mxAlloc==0 ); if( p->nChar+N >= p->nAlloc ){ enlargeAndAppend(p,z,N); }else if( N ){@@ -28295,6 +28723,43 @@ sqlite3TreeViewPop(pView);
} } +/* +** Generate a human-readable description of a SrcList object. +*/ +SQLITE_PRIVATE void sqlite3TreeViewSrcList(TreeView *pView, const SrcList *pSrc){ + int i; + for(i=0; i<pSrc->nSrc; i++){ + const struct SrcList_item *pItem = &pSrc->a[i]; + StrAccum x; + char zLine[100]; + sqlite3StrAccumInit(&x, 0, zLine, sizeof(zLine), 0); + sqlite3_str_appendf(&x, "{%d,*}", pItem->iCursor); + if( pItem->zDatabase ){ + sqlite3_str_appendf(&x, " %s.%s", pItem->zDatabase, pItem->zName); + }else if( pItem->zName ){ + sqlite3_str_appendf(&x, " %s", pItem->zName); + } + if( pItem->pTab ){ + sqlite3_str_appendf(&x, " tab=%Q nCol=%d ptr=%p", + pItem->pTab->zName, pItem->pTab->nCol, pItem->pTab); + } + if( pItem->zAlias ){ + sqlite3_str_appendf(&x, " (AS %s)", pItem->zAlias); + } + if( pItem->fg.jointype & JT_LEFT ){ + sqlite3_str_appendf(&x, " LEFT-JOIN"); + } + sqlite3StrAccumFinish(&x); + sqlite3TreeViewItem(pView, zLine, i<pSrc->nSrc-1); + if( pItem->pSelect ){ + sqlite3TreeViewSelect(pView, pItem->pSelect, 0); + } + if( pItem->fg.isTabFunc ){ + sqlite3TreeViewExprList(pView, pItem->u1.pFuncArg, 0, "func-args:"); + } + sqlite3TreeViewPop(pView); + } +} /* ** Generate a human-readable description of a Select object.@@ -28349,39 +28814,9 @@ sqlite3TreeViewPop(pView);
} #endif if( p->pSrc && p->pSrc->nSrc ){ - int i; pView = sqlite3TreeViewPush(pView, (n--)>0); sqlite3TreeViewLine(pView, "FROM"); - for(i=0; i<p->pSrc->nSrc; i++){ - struct SrcList_item *pItem = &p->pSrc->a[i]; - StrAccum x; - char zLine[100]; - sqlite3StrAccumInit(&x, 0, zLine, sizeof(zLine), 0); - sqlite3_str_appendf(&x, "{%d,*}", pItem->iCursor); - if( pItem->zDatabase ){ - sqlite3_str_appendf(&x, " %s.%s", pItem->zDatabase, pItem->zName); - }else if( pItem->zName ){ - sqlite3_str_appendf(&x, " %s", pItem->zName); - } - if( pItem->pTab ){ - sqlite3_str_appendf(&x, " tabname=%Q", pItem->pTab->zName); - } - if( pItem->zAlias ){ - sqlite3_str_appendf(&x, " (AS %s)", pItem->zAlias); - } - if( pItem->fg.jointype & JT_LEFT ){ - sqlite3_str_appendf(&x, " LEFT-JOIN"); - } - sqlite3StrAccumFinish(&x); - sqlite3TreeViewItem(pView, zLine, i<p->pSrc->nSrc-1); - if( pItem->pSelect ){ - sqlite3TreeViewSelect(pView, pItem->pSelect, 0); - } - if( pItem->fg.isTabFunc ){ - sqlite3TreeViewExprList(pView, pItem->u1.pFuncArg, 0, "func-args:"); - } - sqlite3TreeViewPop(pView); - } + sqlite3TreeViewSrcList(pView, p->pSrc); sqlite3TreeViewPop(pView); } if( p->pWhere ){@@ -28476,22 +28911,60 @@ /*
** Generate a human-readable explanation for a Window object */ SQLITE_PRIVATE void sqlite3TreeViewWindow(TreeView *pView, const Window *pWin, u8 more){ + int nElement = 0; + if( pWin->pFilter ){ + sqlite3TreeViewItem(pView, "FILTER", 1); + sqlite3TreeViewExpr(pView, pWin->pFilter, 0); + sqlite3TreeViewPop(pView); + } pView = sqlite3TreeViewPush(pView, more); if( pWin->zName ){ - sqlite3TreeViewLine(pView, "OVER %s", pWin->zName); + sqlite3TreeViewLine(pView, "OVER %s (%p)", pWin->zName, pWin); }else{ - sqlite3TreeViewLine(pView, "OVER"); + sqlite3TreeViewLine(pView, "OVER (%p)", pWin); + } + if( pWin->zBase ) nElement++; + if( pWin->pOrderBy ) nElement++; + if( pWin->eFrmType ) nElement++; + if( pWin->eExclude ) nElement++; + if( pWin->zBase ){ + sqlite3TreeViewPush(pView, (--nElement)>0); + sqlite3TreeViewLine(pView, "window: %s", pWin->zBase); + sqlite3TreeViewPop(pView); } if( pWin->pPartition ){ - sqlite3TreeViewExprList(pView, pWin->pPartition, 1, "PARTITION-BY"); + sqlite3TreeViewExprList(pView, pWin->pPartition, nElement>0,"PARTITION-BY"); } if( pWin->pOrderBy ){ - sqlite3TreeViewExprList(pView, pWin->pOrderBy, 1, "ORDER-BY"); + sqlite3TreeViewExprList(pView, pWin->pOrderBy, (--nElement)>0, "ORDER-BY"); } - if( pWin->eType ){ - sqlite3TreeViewItem(pView, pWin->eType==TK_RANGE ? "RANGE" : "ROWS", 0); + if( pWin->eFrmType ){ + char zBuf[30]; + const char *zFrmType = "ROWS"; + if( pWin->eFrmType==TK_RANGE ) zFrmType = "RANGE"; + if( pWin->eFrmType==TK_GROUPS ) zFrmType = "GROUPS"; + sqlite3_snprintf(sizeof(zBuf),zBuf,"%s%s",zFrmType, + pWin->bImplicitFrame ? " (implied)" : ""); + sqlite3TreeViewItem(pView, zBuf, (--nElement)>0); sqlite3TreeViewBound(pView, pWin->eStart, pWin->pStart, 1); sqlite3TreeViewBound(pView, pWin->eEnd, pWin->pEnd, 0); + sqlite3TreeViewPop(pView); + } + if( pWin->eExclude ){ + char zBuf[30]; + const char *zExclude; + switch( pWin->eExclude ){ + case TK_NO: zExclude = "NO OTHERS"; break; + case TK_CURRENT: zExclude = "CURRENT ROW"; break; + case TK_GROUP: zExclude = "GROUP"; break; + case TK_TIES: zExclude = "TIES"; break; + default: + sqlite3_snprintf(sizeof(zBuf),zBuf,"invalid(%d)", pWin->eExclude); + zExclude = zBuf; + break; + } + sqlite3TreeViewPush(pView, 0); + sqlite3TreeViewLine(pView, "EXCLUDE %s", zExclude); sqlite3TreeViewPop(pView); } sqlite3TreeViewPop(pView);@@ -28643,7 +29116,7 @@ "IS-FALSE", "IS-TRUE", "IS-NOT-FALSE", "IS-NOT-TRUE"
}; assert( pExpr->op2==TK_IS || pExpr->op2==TK_ISNOT ); assert( pExpr->pRight ); - assert( pExpr->pRight->op==TK_TRUEFALSE ); + assert( sqlite3ExprSkipCollate(pExpr->pRight)->op==TK_TRUEFALSE ); x = (pExpr->op2==TK_ISNOT)*2 + sqlite3ExprTruthValue(pExpr->pRight); zUniOp = azOp[x]; break;@@ -28671,7 +29144,7 @@ pWin = 0;
}else{ pFarg = pExpr->x.pList; #ifndef SQLITE_OMIT_WINDOWFUNC - pWin = pExpr->pWin; + pWin = pExpr->y.pWin; #else pWin = 0; #endif@@ -29473,11 +29946,11 @@ ** desiredEnc. It is an error if the string is already of the desired
** encoding, or if *pMem does not contain a string value. */ SQLITE_PRIVATE SQLITE_NOINLINE int sqlite3VdbeMemTranslate(Mem *pMem, u8 desiredEnc){ - int len; /* Maximum length of output string in bytes */ - unsigned char *zOut; /* Output buffer */ - unsigned char *zIn; /* Input iterator */ - unsigned char *zTerm; /* End of input */ - unsigned char *z; /* Output iterator */ + sqlite3_int64 len; /* Maximum length of output string in bytes */ + unsigned char *zOut; /* Output buffer */ + unsigned char *zIn; /* Input iterator */ + unsigned char *zTerm; /* End of input */ + unsigned char *z; /* Output iterator */ unsigned int c; assert( pMem->db==0 || sqlite3_mutex_held(pMem->db->mutex) );@@ -29526,14 +29999,14 @@ ** A single byte is required for the output string
** nul-terminator. */ pMem->n &= ~1; - len = pMem->n * 2 + 1; + len = 2 * (sqlite3_int64)pMem->n + 1; }else{ /* When converting from UTF-8 to UTF-16 the maximum growth is caused ** when a 1-byte UTF-8 character is translated into a 2-byte UTF-16 ** character. Two bytes are required in the output buffer for the ** nul-terminator. */ - len = pMem->n * 2 + 2; + len = 2 * (sqlite3_int64)pMem->n + 2; } /* Set zIn to point at the start of the input buffer and zTerm to point 1@@ -29825,9 +30298,7 @@ **
*/ /* #include "sqliteInt.h" */ /* #include <stdarg.h> */ -#if HAVE_ISNAN || SQLITE_HAVE_ISNAN -# include <math.h> -#endif +#include <math.h> /* ** Routine needed to support the testcase() macro.@@ -29840,15 +30311,23 @@ }
#endif /* -** Give a callback to the test harness that can be used to simulate faults -** in places where it is difficult or expensive to do so purely by means -** of inputs. +** Calls to sqlite3FaultSim() are used to simulate a failure during testing, +** or to bypass normal error detection during testing in order to let +** execute proceed futher downstream. ** -** The intent of the integer argument is to let the fault simulator know -** which of multiple sqlite3FaultSim() calls has been hit. +** In deployment, sqlite3FaultSim() *always* return SQLITE_OK (0). The +** sqlite3FaultSim() function only returns non-zero during testing. ** -** Return whatever integer value the test callback returns, or return -** SQLITE_OK if no test callback is installed. +** During testing, if the test harness has set a fault-sim callback using +** a call to sqlite3_test_control(SQLITE_TESTCTRL_FAULT_INSTALL), then +** each call to sqlite3FaultSim() is relayed to that application-supplied +** callback and the integer return value form the application-supplied +** callback is returned by sqlite3FaultSim(). +** +** The integer argument to sqlite3FaultSim() is a code to identify which +** sqlite3FaultSim() instance is being invoked. Each call to sqlite3FaultSim() +** should have a unique code. To prevent legacy testing applications from +** breaking, the codes should not be changed or reused. */ #ifndef SQLITE_UNTESTABLE SQLITE_PRIVATE int sqlite3FaultSim(int iTest){@@ -29860,47 +30339,11 @@
#ifndef SQLITE_OMIT_FLOATING_POINT /* ** Return true if the floating point value is Not a Number (NaN). -** -** Use the math library isnan() function if compiled with SQLITE_HAVE_ISNAN. -** Otherwise, we have our own implementation that works on most systems. */ SQLITE_PRIVATE int sqlite3IsNaN(double x){ - int rc; /* The value return */ -#if !SQLITE_HAVE_ISNAN && !HAVE_ISNAN - /* - ** Systems that support the isnan() library function should probably - ** make use of it by compiling with -DSQLITE_HAVE_ISNAN. But we have - ** found that many systems do not have a working isnan() function so - ** this implementation is provided as an alternative. - ** - ** This NaN test sometimes fails if compiled on GCC with -ffast-math. - ** On the other hand, the use of -ffast-math comes with the following - ** warning: - ** - ** This option [-ffast-math] should never be turned on by any - ** -O option since it can result in incorrect output for programs - ** which depend on an exact implementation of IEEE or ISO - ** rules/specifications for math functions. - ** - ** Under MSVC, this NaN test may fail if compiled with a floating- - ** point precision mode other than /fp:precise. From the MSDN - ** documentation: - ** - ** The compiler [with /fp:precise] will properly handle comparisons - ** involving NaN. For example, x != x evaluates to true if x is NaN - ** ... - */ -#ifdef __FAST_MATH__ -# error SQLite will not work correctly with the -ffast-math option of GCC. -#endif - volatile double y = x; - volatile double z = y; - rc = (y!=z); -#else /* if HAVE_ISNAN */ - rc = isnan(x); -#endif /* HAVE_ISNAN */ - testcase( rc ); - return rc; + u64 y; + memcpy(&y,&x,sizeof(y)); + return IsNaN(y); } #endif /* SQLITE_OMIT_FLOATING_POINT */@@ -30034,6 +30477,19 @@ }
} /* +** If database connection db is currently parsing SQL, then transfer +** error code errCode to that parser if the parser has not already +** encountered some other kind of error. +*/ +SQLITE_PRIVATE int sqlite3ErrorToParser(sqlite3 *db, int errCode){ + Parse *pParse; + if( db==0 || (pParse = db->pParse)==0 ) return errCode; + pParse->rc = errCode; + pParse->nErr++; + return errCode; +} + +/* ** Convert an SQL-style quoted string into a normal string by removing ** the quote characters. The conversion is done in-place. If the ** input does not begin with a quote character, then this routine@@ -30046,7 +30502,7 @@ ** The return value is -1 if no dequoting occurs or the length of the
** dequoted string, exclusive of the zero terminator, if dequoting does ** occur. ** -** 2002-Feb-14: This routine is extended to remove MS-Access style +** 2002-02-14: This routine is extended to remove MS-Access style ** brackets from around identifiers. For example: "[a-b-c]" becomes ** "a-b-c". */@@ -30072,6 +30528,11 @@ }
} z[j] = 0; } +SQLITE_PRIVATE void sqlite3DequoteExpr(Expr *p){ + assert( sqlite3Isquote(p->u.zToken[0]) ); + p->flags |= p->u.zToken[0]=='"' ? EP_Quoted|EP_DblQuoted : EP_Quoted; + sqlite3Dequote(p->u.zToken); +} /* ** Generate a Token object from a string@@ -30104,12 +30565,18 @@ return sqlite3StrICmp(zLeft, zRight);
} SQLITE_PRIVATE int sqlite3StrICmp(const char *zLeft, const char *zRight){ unsigned char *a, *b; - int c; + int c, x; a = (unsigned char *)zLeft; b = (unsigned char *)zRight; for(;;){ - c = (int)UpperToLower[*a] - (int)UpperToLower[*b]; - if( c || *a==0 ) break; + c = *a; + x = *b; + if( c==x ){ + if( c==0 ) break; + }else{ + c = (int)UpperToLower[c] - (int)UpperToLower[x]; + if( c ) break; + } a++; b++; }@@ -30137,15 +30604,15 @@ */
static LONGDOUBLE_TYPE sqlite3Pow10(int E){ #if defined(_MSC_VER) static const LONGDOUBLE_TYPE x[] = { - 1.0e+001, - 1.0e+002, - 1.0e+004, - 1.0e+008, - 1.0e+016, - 1.0e+032, - 1.0e+064, - 1.0e+128, - 1.0e+256 + 1.0e+001L, + 1.0e+002L, + 1.0e+004L, + 1.0e+008L, + 1.0e+016L, + 1.0e+032L, + 1.0e+064L, + 1.0e+128L, + 1.0e+256L }; LONGDOUBLE_TYPE r = 1.0; int i;@@ -30175,8 +30642,15 @@ ** The string z[] is length bytes in length (bytes, not characters) and
** uses the encoding enc. The string is not necessarily zero-terminated. ** ** Return TRUE if the result is a valid real number (or integer) and FALSE -** if the string is empty or contains extraneous text. Valid numbers -** are in one of these formats: +** if the string is empty or contains extraneous text. More specifically +** return +** 1 => The input string is a pure integer +** 2 or more => The input has a decimal point or eNNN clause +** 0 or less => The input string is not a valid number +** -1 => Not a valid number, but has a valid prefix which +** includes a decimal point and/or an eNNN clause +** +** Valid numbers are in one of these formats: ** ** [+-]digits[E[+-]digits] ** [+-]digits.[digits][E[+-]digits]@@ -30201,8 +30675,8 @@ int esign = 1; /* sign of exponent */
int e = 0; /* exponent */ int eValid = 1; /* True exponent is either not used or is well-formed */ double result; - int nDigits = 0; - int nonNum = 0; /* True if input contains UTF16 with high byte non-zero */ + int nDigit = 0; /* Number of digits processed */ + int eType = 1; /* 1: pure integer, 2+: fractional -1 or less: bad UTF16 */ assert( enc==SQLITE_UTF8 || enc==SQLITE_UTF16LE || enc==SQLITE_UTF16BE ); *pResult = 0.0; /* Default return value, in case of an error */@@ -30213,8 +30687,10 @@ }else{
int i; incr = 2; assert( SQLITE_UTF16LE==2 && SQLITE_UTF16BE==3 ); + testcase( enc==SQLITE_UTF16LE ); + testcase( enc==SQLITE_UTF16BE ); for(i=3-enc; i<length && z[i]==0; i+=2){} - nonNum = i<length; + if( i<length ) eType = -100; zEnd = &z[i^1]; z += (enc&1); }@@ -30232,27 +30708,30 @@ z+=incr;
} /* copy max significant digits to significand */ - while( z<zEnd && sqlite3Isdigit(*z) && s<((LARGEST_INT64-9)/10) ){ + while( z<zEnd && sqlite3Isdigit(*z) ){ s = s*10 + (*z - '0'); - z+=incr; nDigits++; + z+=incr; nDigit++; + if( s>=((LARGEST_INT64-9)/10) ){ + /* skip non-significant significand digits + ** (increase exponent by d to shift decimal left) */ + while( z<zEnd && sqlite3Isdigit(*z) ){ z+=incr; d++; } + } } - - /* skip non-significant significand digits - ** (increase exponent by d to shift decimal left) */ - while( z<zEnd && sqlite3Isdigit(*z) ){ z+=incr; nDigits++; d++; } if( z>=zEnd ) goto do_atof_calc; /* if decimal point is present */ if( *z=='.' ){ z+=incr; + eType++; /* copy digits from after decimal to significand ** (decrease exponent by d to shift decimal right) */ while( z<zEnd && sqlite3Isdigit(*z) ){ if( s<((LARGEST_INT64-9)/10) ){ s = s*10 + (*z - '0'); d--; + nDigit++; } - z+=incr; nDigits++; + z+=incr; } } if( z>=zEnd ) goto do_atof_calc;@@ -30261,6 +30740,7 @@ /* if exponent is present */
if( *z=='e' || *z=='E' ){ z+=incr; eValid = 0; + eType++; /* This branch is needed to avoid a (harmless) buffer overread. The ** special comment alerts the mutation tester that the correct answer@@ -30359,7 +30839,13 @@ /* store the result */
*pResult = result; /* return true if number and no extra non-whitespace chracters after */ - return z==zEnd && nDigits>0 && eValid && nonNum==0; + if( z==zEnd && nDigit>0 && eValid && eType>0 ){ + return eType; + }else if( eType>=2 && (eType==3 || eValid) && nDigit>0 ){ + return -1; + }else{ + return 0; + } #else return !sqlite3Atoi64(z, pResult, length, enc); #endif /* SQLITE_OMIT_FLOATING_POINT */@@ -30402,6 +30888,7 @@ ** routine does *not* accept hexadecimal notation.
** ** Returns: ** +** -1 Not even a prefix of the input text looks like an integer ** 0 Successful transformation. Fits in a 64-bit signed integer. ** 1 Excess non-space text after the integer value ** 2 Integer too large for a 64-bit signed integer or is malformed@@ -30461,9 +30948,9 @@ }else{
*pNum = (i64)u; } rc = 0; - if( (i==0 && zStart==zNum) /* No digits */ - || nonNum /* UTF16 with high-order bytes non-zero */ - ){ + if( i==0 && zStart==zNum ){ /* No digits */ + rc = -1; + }else if( nonNum ){ /* UTF16 with high-order bytes non-zero */ rc = 1; }else if( &zNum[i]<zEnd ){ /* Extra bytes at the end */ int jj = i;@@ -30694,23 +31181,12 @@ */
SQLITE_PRIVATE u8 sqlite3GetVarint(const unsigned char *p, u64 *v){ u32 a,b,s; - a = *p; - /* a: p0 (unmasked) */ - if (!(a&0x80)) - { - *v = a; + if( ((signed char*)p)[0]>=0 ){ + *v = *p; return 1; } - - p++; - b = *p; - /* b: p1 (unmasked) */ - if (!(b&0x80)) - { - a &= 0x7f; - a = a<<7; - a |= b; - *v = a; + if( ((signed char*)p)[1]>=0 ){ + *v = ((u32)(p[0]&0x7f)<<7) | p[1]; return 2; }@@ -30718,8 +31194,9 @@ /* Verify that constants are precomputed correctly */
assert( SLOT_2_0 == ((0x7f<<14) | (0x7f)) ); assert( SLOT_4_2_0 == ((0xfU<<28) | (0x7f<<14) | (0x7f)) ); - p++; - a = a<<14; + a = ((u32)p[0])<<14; + b = p[1]; + p += 2; a |= *p; /* a: p0<<14 | p2 (unmasked) */ if (!(a&0x80))@@ -31379,7 +31856,7 @@ nInt = nName/4 + 3;
assert( pIn==0 || pIn[0]>=3 ); /* Verify ok to add new elements */ if( pIn==0 || pIn[1]+nInt > pIn[0] ){ /* Enlarge the allocation */ - int nAlloc = (pIn ? pIn[0]*2 : 10) + nInt; + sqlite3_int64 nAlloc = (pIn ? 2*(sqlite3_int64)pIn[0] : 10) + nInt; VList *pOut = sqlite3DbRealloc(db, pIn, nAlloc*sizeof(int)); if( pOut==0 ) return pIn; if( pIn==0 ) pOut[1] = 2;@@ -31585,7 +32062,7 @@ const char *pKey, /* The key we are searching for */
unsigned int *pHash /* Write the hash value here */ ){ HashElem *elem; /* Used to loop thru the element list */ - int count; /* Number of elements left to test */ + unsigned int count; /* Number of elements left to test */ unsigned int h; /* The computed hash */ static HashElem nullElement = { 0, 0, 0, 0 };@@ -31633,8 +32110,8 @@ pEntry = &pH->ht[h];
if( pEntry->chain==elem ){ pEntry->chain = elem->next; } + assert( pEntry->count>0 ); pEntry->count--; - assert( pEntry->count>=0 ); } sqlite3_free( elem ); pH->count--;@@ -31809,25 +32286,25 @@ /* 88 */ "IsTrue" OpHelp("r[P2] = coalesce(r[P1]==TRUE,P3) ^ P4"),
/* 89 */ "Offset" OpHelp("r[P3] = sqlite_offset(P1)"), /* 90 */ "Column" OpHelp("r[P3]=PX"), /* 91 */ "Affinity" OpHelp("affinity(r[P1@P2])"), - /* 92 */ "BitAnd" OpHelp("r[P3]=r[P1]&r[P2]"), - /* 93 */ "BitOr" OpHelp("r[P3]=r[P1]|r[P2]"), - /* 94 */ "ShiftLeft" OpHelp("r[P3]=r[P2]<<r[P1]"), - /* 95 */ "ShiftRight" OpHelp("r[P3]=r[P2]>>r[P1]"), - /* 96 */ "Add" OpHelp("r[P3]=r[P1]+r[P2]"), - /* 97 */ "Subtract" OpHelp("r[P3]=r[P2]-r[P1]"), - /* 98 */ "Multiply" OpHelp("r[P3]=r[P1]*r[P2]"), - /* 99 */ "Divide" OpHelp("r[P3]=r[P2]/r[P1]"), - /* 100 */ "Remainder" OpHelp("r[P3]=r[P2]%r[P1]"), - /* 101 */ "Concat" OpHelp("r[P3]=r[P2]+r[P1]"), - /* 102 */ "MakeRecord" OpHelp("r[P3]=mkrec(r[P1@P2])"), - /* 103 */ "BitNot" OpHelp("r[P2]= ~r[P1]"), - /* 104 */ "Count" OpHelp("r[P2]=count()"), - /* 105 */ "ReadCookie" OpHelp(""), - /* 106 */ "String8" OpHelp("r[P2]='P4'"), - /* 107 */ "SetCookie" OpHelp(""), - /* 108 */ "ReopenIdx" OpHelp("root=P2 iDb=P3"), - /* 109 */ "OpenRead" OpHelp("root=P2 iDb=P3"), - /* 110 */ "OpenWrite" OpHelp("root=P2 iDb=P3"), + /* 92 */ "MakeRecord" OpHelp("r[P3]=mkrec(r[P1@P2])"), + /* 93 */ "Count" OpHelp("r[P2]=count()"), + /* 94 */ "ReadCookie" OpHelp(""), + /* 95 */ "SetCookie" OpHelp(""), + /* 96 */ "BitAnd" OpHelp("r[P3]=r[P1]&r[P2]"), + /* 97 */ "BitOr" OpHelp("r[P3]=r[P1]|r[P2]"), + /* 98 */ "ShiftLeft" OpHelp("r[P3]=r[P2]<<r[P1]"), + /* 99 */ "ShiftRight" OpHelp("r[P3]=r[P2]>>r[P1]"), + /* 100 */ "Add" OpHelp("r[P3]=r[P1]+r[P2]"), + /* 101 */ "Subtract" OpHelp("r[P3]=r[P2]-r[P1]"), + /* 102 */ "Multiply" OpHelp("r[P3]=r[P1]*r[P2]"), + /* 103 */ "Divide" OpHelp("r[P3]=r[P2]/r[P1]"), + /* 104 */ "Remainder" OpHelp("r[P3]=r[P2]%r[P1]"), + /* 105 */ "Concat" OpHelp("r[P3]=r[P2]+r[P1]"), + /* 106 */ "ReopenIdx" OpHelp("root=P2 iDb=P3"), + /* 107 */ "BitNot" OpHelp("r[P2]= ~r[P1]"), + /* 108 */ "OpenRead" OpHelp("root=P2 iDb=P3"), + /* 109 */ "OpenWrite" OpHelp("root=P2 iDb=P3"), + /* 110 */ "String8" OpHelp("r[P2]='P4'"), /* 111 */ "OpenDup" OpHelp(""), /* 112 */ "OpenAutoindex" OpHelp("nColumn=P2"), /* 113 */ "OpenEphemeral" OpHelp("nColumn=P2"),@@ -31840,57 +32317,56 @@ /* 119 */ "SeekHit" OpHelp("seekHit=P2"),
/* 120 */ "Sequence" OpHelp("r[P2]=cursor[P1].ctr++"), /* 121 */ "NewRowid" OpHelp("r[P2]=rowid"), /* 122 */ "Insert" OpHelp("intkey=r[P3] data=r[P2]"), - /* 123 */ "InsertInt" OpHelp("intkey=P3 data=r[P2]"), - /* 124 */ "Delete" OpHelp(""), - /* 125 */ "ResetCount" OpHelp(""), - /* 126 */ "SorterCompare" OpHelp("if key(P1)!=trim(r[P3],P4) goto P2"), - /* 127 */ "SorterData" OpHelp("r[P2]=data"), - /* 128 */ "RowData" OpHelp("r[P2]=data"), - /* 129 */ "Rowid" OpHelp("r[P2]=rowid"), - /* 130 */ "NullRow" OpHelp(""), - /* 131 */ "SeekEnd" OpHelp(""), - /* 132 */ "SorterInsert" OpHelp("key=r[P2]"), - /* 133 */ "IdxInsert" OpHelp("key=r[P2]"), - /* 134 */ "IdxDelete" OpHelp("key=r[P2@P3]"), - /* 135 */ "DeferredSeek" OpHelp("Move P3 to P1.rowid if needed"), - /* 136 */ "IdxRowid" OpHelp("r[P2]=rowid"), - /* 137 */ "Destroy" OpHelp(""), - /* 138 */ "Clear" OpHelp(""), - /* 139 */ "ResetSorter" OpHelp(""), - /* 140 */ "CreateBtree" OpHelp("r[P2]=root iDb=P1 flags=P3"), - /* 141 */ "Real" OpHelp("r[P2]=P4"), - /* 142 */ "SqlExec" OpHelp(""), - /* 143 */ "ParseSchema" OpHelp(""), - /* 144 */ "LoadAnalysis" OpHelp(""), - /* 145 */ "DropTable" OpHelp(""), - /* 146 */ "DropIndex" OpHelp(""), - /* 147 */ "DropTrigger" OpHelp(""), - /* 148 */ "IntegrityCk" OpHelp(""), - /* 149 */ "RowSetAdd" OpHelp("rowset(P1)=r[P2]"), - /* 150 */ "Param" OpHelp(""), - /* 151 */ "FkCounter" OpHelp("fkctr[P1]+=P2"), - /* 152 */ "MemMax" OpHelp("r[P1]=max(r[P1],r[P2])"), - /* 153 */ "OffsetLimit" OpHelp("if r[P1]>0 then r[P2]=r[P1]+max(0,r[P3]) else r[P2]=(-1)"), - /* 154 */ "AggInverse" OpHelp("accum=r[P3] inverse(r[P2@P5])"), - /* 155 */ "AggStep" OpHelp("accum=r[P3] step(r[P2@P5])"), - /* 156 */ "AggStep1" OpHelp("accum=r[P3] step(r[P2@P5])"), - /* 157 */ "AggValue" OpHelp("r[P3]=value N=P2"), - /* 158 */ "AggFinal" OpHelp("accum=r[P1] N=P2"), - /* 159 */ "Expire" OpHelp(""), - /* 160 */ "TableLock" OpHelp("iDb=P1 root=P2 write=P3"), - /* 161 */ "VBegin" OpHelp(""), - /* 162 */ "VCreate" OpHelp(""), - /* 163 */ "VDestroy" OpHelp(""), - /* 164 */ "VOpen" OpHelp(""), - /* 165 */ "VColumn" OpHelp("r[P3]=vcolumn(P2)"), - /* 166 */ "VRename" OpHelp(""), - /* 167 */ "Pagecount" OpHelp(""), - /* 168 */ "MaxPgcnt" OpHelp(""), - /* 169 */ "Trace" OpHelp(""), - /* 170 */ "CursorHint" OpHelp(""), - /* 171 */ "Noop" OpHelp(""), - /* 172 */ "Explain" OpHelp(""), - /* 173 */ "Abortable" OpHelp(""), + /* 123 */ "Delete" OpHelp(""), + /* 124 */ "ResetCount" OpHelp(""), + /* 125 */ "SorterCompare" OpHelp("if key(P1)!=trim(r[P3],P4) goto P2"), + /* 126 */ "SorterData" OpHelp("r[P2]=data"), + /* 127 */ "RowData" OpHelp("r[P2]=data"), + /* 128 */ "Rowid" OpHelp("r[P2]=rowid"), + /* 129 */ "NullRow" OpHelp(""), + /* 130 */ "SeekEnd" OpHelp(""), + /* 131 */ "SorterInsert" OpHelp("key=r[P2]"), + /* 132 */ "IdxInsert" OpHelp("key=r[P2]"), + /* 133 */ "IdxDelete" OpHelp("key=r[P2@P3]"), + /* 134 */ "DeferredSeek" OpHelp("Move P3 to P1.rowid if needed"), + /* 135 */ "IdxRowid" OpHelp("r[P2]=rowid"), + /* 136 */ "Destroy" OpHelp(""), + /* 137 */ "Clear" OpHelp(""), + /* 138 */ "ResetSorter" OpHelp(""), + /* 139 */ "CreateBtree" OpHelp("r[P2]=root iDb=P1 flags=P3"), + /* 140 */ "SqlExec" OpHelp(""), + /* 141 */ "ParseSchema" OpHelp(""), + /* 142 */ "LoadAnalysis" OpHelp(""), + /* 143 */ "DropTable" OpHelp(""), + /* 144 */ "DropIndex" OpHelp(""), + /* 145 */ "Real" OpHelp("r[P2]=P4"), + /* 146 */ "DropTrigger" OpHelp(""), + /* 147 */ "IntegrityCk" OpHelp(""), + /* 148 */ "RowSetAdd" OpHelp("rowset(P1)=r[P2]"), + /* 149 */ "Param" OpHelp(""), + /* 150 */ "FkCounter" OpHelp("fkctr[P1]+=P2"), + /* 151 */ "MemMax" OpHelp("r[P1]=max(r[P1],r[P2])"), + /* 152 */ "OffsetLimit" OpHelp("if r[P1]>0 then r[P2]=r[P1]+max(0,r[P3]) else r[P2]=(-1)"), + /* 153 */ "AggInverse" OpHelp("accum=r[P3] inverse(r[P2@P5])"), + /* 154 */ "AggStep" OpHelp("accum=r[P3] step(r[P2@P5])"), + /* 155 */ "AggStep1" OpHelp("accum=r[P3] step(r[P2@P5])"), + /* 156 */ "AggValue" OpHelp("r[P3]=value N=P2"), + /* 157 */ "AggFinal" OpHelp("accum=r[P1] N=P2"), + /* 158 */ "Expire" OpHelp(""), + /* 159 */ "TableLock" OpHelp("iDb=P1 root=P2 write=P3"), + /* 160 */ "VBegin" OpHelp(""), + /* 161 */ "VCreate" OpHelp(""), + /* 162 */ "VDestroy" OpHelp(""), + /* 163 */ "VOpen" OpHelp(""), + /* 164 */ "VColumn" OpHelp("r[P3]=vcolumn(P2)"), + /* 165 */ "VRename" OpHelp(""), + /* 166 */ "Pagecount" OpHelp(""), + /* 167 */ "MaxPgcnt" OpHelp(""), + /* 168 */ "Trace" OpHelp(""), + /* 169 */ "CursorHint" OpHelp(""), + /* 170 */ "Noop" OpHelp(""), + /* 171 */ "Explain" OpHelp(""), + /* 172 */ "Abortable" OpHelp(""), }; return azName[i]; }@@ -32036,12 +32512,10 @@ */
#define SQLITE_FSFLAGS_IS_MSDOS 0x1 /* -** If we are to be thread-safe, include the pthreads header and define -** the SQLITE_UNIX_THREADS macro. +** If we are to be thread-safe, include the pthreads header. */ #if SQLITE_THREADSAFE /* # include <pthread.h> */ -# define SQLITE_UNIX_THREADS 1 #endif /*@@ -32619,7 +33093,11 @@ #endif
#define osLstat ((int(*)(const char*,struct stat*))aSyscall[27].pCurrent) #if defined(__linux__) && defined(SQLITE_ENABLE_BATCH_ATOMIC_WRITE) +# ifdef __ANDROID__ + { "ioctl", (sqlite3_syscall_ptr)(int(*)(int, int, ...))ioctl, 0 }, +# else { "ioctl", (sqlite3_syscall_ptr)ioctl, 0 }, +# endif #else { "ioctl", (sqlite3_syscall_ptr)0, 0 }, #endif@@ -33213,8 +33691,7 @@ };
/* ** An instance of the following structure is allocated for each open -** inode. Or, on LinuxThreads, there is one of these structures for -** each inode opened by each thread. +** inode. ** ** A single inode can have multiple file descriptors, so each unixFile ** structure contains a pointer to an instance of this object and this@@ -33260,13 +33737,16 @@ };
/* ** A lists of all unixInodeInfo objects. +** +** Must hold unixBigLock in order to read or write this variable. */ static unixInodeInfo *inodeList = 0; /* All unixInodeInfo objects */ #ifdef SQLITE_DEBUG /* -** True if the inode mutex is held, or not. Used only within assert() -** to help verify correct mutex usage. +** True if the inode mutex (on the unixFile.pFileMutex field) is held, or not. +** This routine is used only within assert() to help verify correct mutex +** usage. */ int unixFileMutexHeld(unixFile *pFile){ assert( pFile->pInode );@@ -33394,8 +33874,8 @@
/* ** Release a unixInodeInfo structure previously allocated by findInodeInfo(). ** -** The mutex entered using the unixEnterMutex() function must be held -** when this function is called. +** The global mutex must be held when this routine is called, but the mutex +** on the inode being deleted must NOT be held. */ static void releaseInodeInfo(unixFile *pFile){ unixInodeInfo *pInode = pFile->pInode;@@ -33430,8 +33910,7 @@ ** Given a file descriptor, locate the unixInodeInfo object that
** describes that file descriptor. Create a new one if necessary. The ** return value might be uninitialized if an error occurs. ** -** The mutex entered using the unixEnterMutex() function must be held -** when this function is called. +** The global mutex must held when calling this routine. ** ** Return an appropriate error code. */@@ -33492,6 +33971,7 @@ fileId.pId = pFile->pId;
#else fileId.ino = (u64)statbuf.st_ino; #endif + assert( unixMutexHeld() ); pInode = inodeList; while( pInode && memcmp(&fileId, &pInode->fileId, sizeof(fileId)) ){ pInode = pInode->pNext;@@ -33511,6 +33991,7 @@ return SQLITE_NOMEM_BKPT;
} } pInode->nRef = 1; + assert( unixMutexHeld() ); pInode->pNext = inodeList; pInode->pPrev = 0; if( inodeList ) inodeList->pPrev = pInode;@@ -36308,18 +36789,18 @@ ** nRef
** ** The following fields are read-only after the object is created: ** -** fid +** hShm ** zFilename ** -** Either unixShmNode.mutex must be held or unixShmNode.nRef==0 and +** Either unixShmNode.pShmMutex must be held or unixShmNode.nRef==0 and ** unixMutexHeld() is true when reading or writing any other field ** in this structure. */ struct unixShmNode { unixInodeInfo *pInode; /* unixInodeInfo that owns this SHM node */ - sqlite3_mutex *mutex; /* Mutex to access this object */ + sqlite3_mutex *pShmMutex; /* Mutex to access this object */ char *zFilename; /* Name of the mmapped file */ - int h; /* Open file descriptor */ + int hShm; /* Open file descriptor */ int szRegion; /* Size of shared-memory regions */ u16 nRegion; /* Size of array apRegion */ u8 isReadonly; /* True if read-only */@@ -36341,16 +36822,16 @@ **
** The following fields are initialized when this object is created and ** are read-only thereafter: ** -** unixShm.pFile +** unixShm.pShmNode ** unixShm.id ** -** All other fields are read/write. The unixShm.pFile->mutex must be held -** while accessing any read/write fields. +** All other fields are read/write. The unixShm.pShmNode->pShmMutex must +** be held while accessing any read/write fields. */ struct unixShm { unixShmNode *pShmNode; /* The underlying unixShmNode object */ unixShm *pNext; /* Next unixShm with the same unixShmNode */ - u8 hasMutex; /* True if holding the unixShmNode mutex */ + u8 hasMutex; /* True if holding the unixShmNode->pShmMutex */ u8 id; /* Id of this connection within its unixShmNode */ u16 sharedMask; /* Mask of shared locks held */ u16 exclMask; /* Mask of exclusive locks held */@@ -36380,7 +36861,8 @@ int rc = SQLITE_OK; /* Result code form fcntl() */
/* Access to the unixShmNode object is serialized by the caller */ pShmNode = pFile->pInode->pShmNode; - assert( pShmNode->nRef==0 || sqlite3_mutex_held(pShmNode->mutex) ); + assert( pShmNode->nRef==0 || sqlite3_mutex_held(pShmNode->pShmMutex) ); + assert( pShmNode->nRef>0 || unixMutexHeld() ); /* Shared locks never span more than one byte */ assert( n==1 || lockType!=F_RDLCK );@@ -36388,13 +36870,13 @@
/* Locks are within range */ assert( n>=1 && n<=SQLITE_SHM_NLOCK ); - if( pShmNode->h>=0 ){ + if( pShmNode->hShm>=0 ){ /* Initialize the locking parameters */ f.l_type = lockType; f.l_whence = SEEK_SET; f.l_start = ofst; f.l_len = n; - rc = osSetPosixAdvisoryLock(pShmNode->h, &f, pFile); + rc = osSetPosixAdvisoryLock(pShmNode->hShm, &f, pFile); rc = (rc!=(-1)) ? SQLITE_OK : SQLITE_BUSY; }@@ -36466,18 +36948,18 @@ if( p && ALWAYS(p->nRef==0) ){
int nShmPerMap = unixShmRegionPerMap(); int i; assert( p->pInode==pFd->pInode ); - sqlite3_mutex_free(p->mutex); + sqlite3_mutex_free(p->pShmMutex); for(i=0; i<p->nRegion; i+=nShmPerMap){ - if( p->h>=0 ){ + if( p->hShm>=0 ){ osMunmap(p->apRegion[i], p->szRegion); }else{ sqlite3_free(p->apRegion[i]); } } sqlite3_free(p->apRegion); - if( p->h>=0 ){ - robust_close(pFd, p->h, __LINE__); - p->h = -1; + if( p->hShm>=0 ){ + robust_close(pFd, p->hShm, __LINE__); + p->hShm = -1; } p->pInode->pShmNode = 0; sqlite3_free(p);@@ -36519,7 +37001,7 @@ lock.l_whence = SEEK_SET;
lock.l_start = UNIX_SHM_DMS; lock.l_len = 1; lock.l_type = F_WRLCK; - if( osFcntl(pShmNode->h, F_GETLK, &lock)!=0 ) { + if( osFcntl(pShmNode->hShm, F_GETLK, &lock)!=0 ) { rc = SQLITE_IOERR_LOCK; }else if( lock.l_type==F_UNLCK ){ if( pShmNode->isReadonly ){@@ -36527,7 +37009,12 @@ pShmNode->isUnlocked = 1;
rc = SQLITE_READONLY_CANTINIT; }else{ rc = unixShmSystemLock(pDbFd, F_WRLCK, UNIX_SHM_DMS, 1); - if( rc==SQLITE_OK && robust_ftruncate(pShmNode->h, 0) ){ + /* The first connection to attach must truncate the -shm file. We + ** truncate to 3 bytes (an arbitrary small number, less than the + ** -shm header size) rather than 0 as a system debugging aid, to + ** help detect if a -shm file truncation is legitimate or is the work + ** or a rogue process. */ + if( rc==SQLITE_OK && robust_ftruncate(pShmNode->hShm, 3) ){ rc = unixLogError(SQLITE_IOERR_SHMOPEN,"ftruncate",pShmNode->zFilename); } }@@ -36633,12 +37120,12 @@ #else
sqlite3_snprintf(nShmFilename, zShm, "%s-shm", zBasePath); sqlite3FileSuffix3(pDbFd->zPath, zShm); #endif - pShmNode->h = -1; + pShmNode->hShm = -1; pDbFd->pInode->pShmNode = pShmNode; pShmNode->pInode = pDbFd->pInode; if( sqlite3GlobalConfig.bCoreMutex ){ - pShmNode->mutex = sqlite3_mutex_alloc(SQLITE_MUTEX_FAST); - if( pShmNode->mutex==0 ){ + pShmNode->pShmMutex = sqlite3_mutex_alloc(SQLITE_MUTEX_FAST); + if( pShmNode->pShmMutex==0 ){ rc = SQLITE_NOMEM_BKPT; goto shm_open_err; }@@ -36646,11 +37133,11 @@ }
if( pInode->bProcessLock==0 ){ if( 0==sqlite3_uri_boolean(pDbFd->zPath, "readonly_shm", 0) ){ - pShmNode->h = robust_open(zShm, O_RDWR|O_CREAT, (sStat.st_mode&0777)); + pShmNode->hShm = robust_open(zShm, O_RDWR|O_CREAT,(sStat.st_mode&0777)); } - if( pShmNode->h<0 ){ - pShmNode->h = robust_open(zShm, O_RDONLY, (sStat.st_mode&0777)); - if( pShmNode->h<0 ){ + if( pShmNode->hShm<0 ){ + pShmNode->hShm = robust_open(zShm, O_RDONLY, (sStat.st_mode&0777)); + if( pShmNode->hShm<0 ){ rc = unixLogError(SQLITE_CANTOPEN_BKPT, "open", zShm); goto shm_open_err; }@@ -36661,7 +37148,7 @@ /* If this process is running as root, make sure that the SHM file
** is owned by the same user that owns the original database. Otherwise, ** the original owner will not be able to connect. */ - robustFchown(pShmNode->h, sStat.st_uid, sStat.st_gid); + robustFchown(pShmNode->hShm, sStat.st_uid, sStat.st_gid); rc = unixLockSharedMemory(pDbFd, pShmNode); if( rc!=SQLITE_OK && rc!=SQLITE_READONLY_CANTINIT ) goto shm_open_err;@@ -36681,13 +37168,13 @@ /* The reference count on pShmNode has already been incremented under
** the cover of the unixEnterMutex() mutex and the pointer from the ** new (struct unixShm) object to the pShmNode has been set. All that is ** left to do is to link the new object into the linked list starting - ** at pShmNode->pFirst. This must be done while holding the pShmNode->mutex - ** mutex. + ** at pShmNode->pFirst. This must be done while holding the + ** pShmNode->pShmMutex. */ - sqlite3_mutex_enter(pShmNode->mutex); + sqlite3_mutex_enter(pShmNode->pShmMutex); p->pNext = pShmNode->pFirst; pShmNode->pFirst = p; - sqlite3_mutex_leave(pShmNode->mutex); + sqlite3_mutex_leave(pShmNode->pShmMutex); return rc; /* Jump here on any error */@@ -36739,7 +37226,7 @@ }
p = pDbFd->pShm; pShmNode = p->pShmNode; - sqlite3_mutex_enter(pShmNode->mutex); + sqlite3_mutex_enter(pShmNode->pShmMutex); if( pShmNode->isUnlocked ){ rc = unixLockSharedMemory(pDbFd, pShmNode); if( rc!=SQLITE_OK ) goto shmpage_out;@@ -36747,8 +37234,8 @@ pShmNode->isUnlocked = 0;
} assert( szRegion==pShmNode->szRegion || pShmNode->nRegion==0 ); assert( pShmNode->pInode==pDbFd->pInode ); - assert( pShmNode->h>=0 || pDbFd->pInode->bProcessLock==1 ); - assert( pShmNode->h<0 || pDbFd->pInode->bProcessLock==0 ); + assert( pShmNode->hShm>=0 || pDbFd->pInode->bProcessLock==1 ); + assert( pShmNode->hShm<0 || pDbFd->pInode->bProcessLock==0 ); /* Minimum number of regions required to be mapped. */ nReqRegion = ((iRegion+nShmPerMap) / nShmPerMap) * nShmPerMap;@@ -36760,12 +37247,12 @@ struct stat sStat; /* Used by fstat() */
pShmNode->szRegion = szRegion; - if( pShmNode->h>=0 ){ + if( pShmNode->hShm>=0 ){ /* The requested region is not mapped into this processes address space. ** Check to see if it has been allocated (i.e. if the wal-index file is ** large enough to contain the requested region). */ - if( osFstat(pShmNode->h, &sStat) ){ + if( osFstat(pShmNode->hShm, &sStat) ){ rc = SQLITE_IOERR_SHMSIZE; goto shmpage_out; }@@ -36793,7 +37280,7 @@ /* Write to the last byte of each newly allocated or extended page */
assert( (nByte % pgsz)==0 ); for(iPg=(sStat.st_size/pgsz); iPg<(nByte/pgsz); iPg++){ int x = 0; - if( seekAndWriteFd(pShmNode->h, iPg*pgsz + pgsz-1, "", 1, &x)!=1 ){ + if( seekAndWriteFd(pShmNode->hShm, iPg*pgsz + pgsz-1,"",1,&x)!=1 ){ const char *zFile = pShmNode->zFilename; rc = unixLogError(SQLITE_IOERR_SHMSIZE, "write", zFile); goto shmpage_out;@@ -36816,22 +37303,22 @@ while( pShmNode->nRegion<nReqRegion ){
int nMap = szRegion*nShmPerMap; int i; void *pMem; - if( pShmNode->h>=0 ){ + if( pShmNode->hShm>=0 ){ pMem = osMmap(0, nMap, pShmNode->isReadonly ? PROT_READ : PROT_READ|PROT_WRITE, - MAP_SHARED, pShmNode->h, szRegion*(i64)pShmNode->nRegion + MAP_SHARED, pShmNode->hShm, szRegion*(i64)pShmNode->nRegion ); if( pMem==MAP_FAILED ){ rc = unixLogError(SQLITE_IOERR_SHMMAP, "mmap", pShmNode->zFilename); goto shmpage_out; } }else{ - pMem = sqlite3_malloc64(szRegion); + pMem = sqlite3_malloc64(nMap); if( pMem==0 ){ rc = SQLITE_NOMEM_BKPT; goto shmpage_out; } - memset(pMem, 0, szRegion); + memset(pMem, 0, nMap); } for(i=0; i<nShmPerMap; i++){@@ -36848,7 +37335,7 @@ }else{
*pp = 0; } if( pShmNode->isReadonly && rc==SQLITE_OK ) rc = SQLITE_READONLY; - sqlite3_mutex_leave(pShmNode->mutex); + sqlite3_mutex_leave(pShmNode->pShmMutex); return rc; }@@ -36882,12 +37369,12 @@ || flags==(SQLITE_SHM_LOCK | SQLITE_SHM_EXCLUSIVE)
|| flags==(SQLITE_SHM_UNLOCK | SQLITE_SHM_SHARED) || flags==(SQLITE_SHM_UNLOCK | SQLITE_SHM_EXCLUSIVE) ); assert( n==1 || (flags & SQLITE_SHM_EXCLUSIVE)!=0 ); - assert( pShmNode->h>=0 || pDbFd->pInode->bProcessLock==1 ); - assert( pShmNode->h<0 || pDbFd->pInode->bProcessLock==0 ); + assert( pShmNode->hShm>=0 || pDbFd->pInode->bProcessLock==1 ); + assert( pShmNode->hShm<0 || pDbFd->pInode->bProcessLock==0 ); mask = (1<<(ofst+n)) - (1<<ofst); assert( n>1 || mask==(1<<ofst) ); - sqlite3_mutex_enter(pShmNode->mutex); + sqlite3_mutex_enter(pShmNode->pShmMutex); if( flags & SQLITE_SHM_UNLOCK ){ u16 allMask = 0; /* Mask of locks held by siblings */@@ -36960,7 +37447,7 @@ p->exclMask |= mask;
} } } - sqlite3_mutex_leave(pShmNode->mutex); + sqlite3_mutex_leave(pShmNode->pShmMutex); OSTRACE(("SHM-LOCK shmid-%d, pid-%d got %03x,%03x\n", p->id, osGetpid(0), p->sharedMask, p->exclMask)); return rc;@@ -37010,14 +37497,14 @@ assert( pShmNode->pInode==pDbFd->pInode );
/* Remove connection p from the set of connections associated ** with pShmNode */ - sqlite3_mutex_enter(pShmNode->mutex); + sqlite3_mutex_enter(pShmNode->pShmMutex); for(pp=&pShmNode->pFirst; (*pp)!=p; pp = &(*pp)->pNext){} *pp = p->pNext; /* Free the connection p */ sqlite3_free(p); pDbFd->pShm = 0; - sqlite3_mutex_leave(pShmNode->mutex); + sqlite3_mutex_leave(pShmNode->pShmMutex); /* If pShmNode->nRef has reached 0, then close the underlying ** shared-memory file, too */@@ -37026,7 +37513,7 @@ unixEnterMutex();
assert( pShmNode->nRef>0 ); pShmNode->nRef--; if( pShmNode->nRef==0 ){ - if( deleteFlag && pShmNode->h>=0 ){ + if( deleteFlag && pShmNode->hShm>=0 ){ osUnlink(pShmNode->zFilename); } unixShmPurge(pDbFd);@@ -40444,8 +40931,7 @@ #if SQLITE_MAX_MMAP_SIZE>0
int nFetchOut; /* Number of outstanding xFetch references */ HANDLE hMap; /* Handle for accessing memory mapping */ void *pMapRegion; /* Area memory mapped */ - sqlite3_int64 mmapSize; /* Usable size of mapped region */ - sqlite3_int64 mmapSizeActual; /* Actual size of mapped region */ + sqlite3_int64 mmapSize; /* Size of mapped region */ sqlite3_int64 mmapSizeMax; /* Configured FCNTL_MMAP_SIZE value */ #endif };@@ -43066,6 +43552,26 @@ int rc = SQLITE_OK; /* Return code for this function */
DWORD lastErrno; #if SQLITE_MAX_MMAP_SIZE>0 sqlite3_int64 oldMmapSize; + if( pFile->nFetchOut>0 ){ + /* File truncation is a no-op if there are outstanding memory mapped + ** pages. This is because truncating the file means temporarily unmapping + ** the file, and that might delete memory out from under existing cursors. + ** + ** This can result in incremental vacuum not truncating the file, + ** if there is an active read cursor when the incremental vacuum occurs. + ** No real harm comes of this - the database file is not corrupted, + ** though some folks might complain that the file is bigger than it + ** needs to be. + ** + ** The only feasible work-around is to defer the truncation until after + ** all references to memory-mapped content are closed. That is doable, + ** but involves adding a few branches in the common write code path which + ** could slow down normal operations slightly. Hence, we have decided for + ** now to simply make trancations a no-op if there are pending reads. We + ** can maybe revisit this decision in the future. + */ + return SQLITE_OK; + } #endif assert( pFile );@@ -44494,9 +45000,9 @@ #if SQLITE_MAX_MMAP_SIZE>0
static int winUnmapfile(winFile *pFile){ assert( pFile!=0 ); OSTRACE(("UNMAP-FILE pid=%lu, pFile=%p, hMap=%p, pMapRegion=%p, " - "mmapSize=%lld, mmapSizeActual=%lld, mmapSizeMax=%lld\n", + "mmapSize=%lld, mmapSizeMax=%lld\n", osGetCurrentProcessId(), pFile, pFile->hMap, pFile->pMapRegion, - pFile->mmapSize, pFile->mmapSizeActual, pFile->mmapSizeMax)); + pFile->mmapSize, pFile->mmapSizeMax)); if( pFile->pMapRegion ){ if( !osUnmapViewOfFile(pFile->pMapRegion) ){ pFile->lastErrno = osGetLastError();@@ -44508,7 +45014,6 @@ "winUnmapfile1", pFile->zPath);
} pFile->pMapRegion = 0; pFile->mmapSize = 0; - pFile->mmapSizeActual = 0; } if( pFile->hMap!=NULL ){ if( !osCloseHandle(pFile->hMap) ){@@ -44619,7 +45124,6 @@ return SQLITE_OK;
} pFd->pMapRegion = pNew; pFd->mmapSize = nMap; - pFd->mmapSizeActual = nMap; } OSTRACE(("MAP-FILE pid=%lu, pFile=%p, rc=SQLITE_OK\n",@@ -45421,7 +45925,6 @@ #if SQLITE_MAX_MMAP_SIZE>0
pFile->hMap = NULL; pFile->pMapRegion = 0; pFile->mmapSize = 0; - pFile->mmapSizeActual = 0; pFile->mmapSizeMax = sqlite3GlobalConfig.szMmap; #endif@@ -46314,7 +46817,8 @@ /* An open file */
struct MemFile { sqlite3_file base; /* IO methods */ sqlite3_int64 sz; /* Size of the file */ - sqlite3_int64 szMax; /* Space allocated to aData */ + sqlite3_int64 szAlloc; /* Space allocated to aData */ + sqlite3_int64 szMax; /* Maximum allowed size of the file */ unsigned char *aData; /* content of the file */ int nMmap; /* Number of memory mapped pages */ unsigned mFlags; /* Flags */@@ -46440,10 +46944,15 @@ unsigned char *pNew;
if( (p->mFlags & SQLITE_DESERIALIZE_RESIZEABLE)==0 || p->nMmap>0 ){ return SQLITE_FULL; } + if( newSz>p->szMax ){ + return SQLITE_FULL; + } + newSz *= 2; + if( newSz>p->szMax ) newSz = p->szMax; pNew = sqlite3_realloc64(p->aData, newSz); if( pNew==0 ) return SQLITE_NOMEM; p->aData = pNew; - p->szMax = newSz; + p->szAlloc = newSz; return SQLITE_OK; }@@ -46457,10 +46966,11 @@ int iAmt,
sqlite_int64 iOfst ){ MemFile *p = (MemFile *)pFile; + if( NEVER(p->mFlags & SQLITE_DESERIALIZE_READONLY) ) return SQLITE_READONLY; if( iOfst+iAmt>p->sz ){ int rc; - if( iOfst+iAmt>p->szMax - && (rc = memdbEnlarge(p, (iOfst+iAmt)*2))!=SQLITE_OK + if( iOfst+iAmt>p->szAlloc + && (rc = memdbEnlarge(p, iOfst+iAmt))!=SQLITE_OK ){ return rc; }@@ -46506,6 +47016,11 @@ ** Lock an memdb-file.
*/ static int memdbLock(sqlite3_file *pFile, int eLock){ MemFile *p = (MemFile *)pFile; + if( eLock>SQLITE_LOCK_SHARED + && (p->mFlags & SQLITE_DESERIALIZE_READONLY)!=0 + ){ + return SQLITE_READONLY; + } p->eLock = eLock; return SQLITE_OK; }@@ -46530,6 +47045,19 @@ if( op==SQLITE_FCNTL_VFSNAME ){
*(char**)pArg = sqlite3_mprintf("memdb(%p,%lld)", p->aData, p->sz); rc = SQLITE_OK; } + if( op==SQLITE_FCNTL_SIZE_LIMIT ){ + sqlite3_int64 iLimit = *(sqlite3_int64*)pArg; + if( iLimit<p->sz ){ + if( iLimit<0 ){ + iLimit = p->szMax; + }else{ + iLimit = p->sz; + } + } + p->szMax = iLimit; + *(sqlite3_int64*)pArg = iLimit; + rc = SQLITE_OK; + } return rc; }@@ -46560,8 +47088,12 @@ int iAmt,
void **pp ){ MemFile *p = (MemFile *)pFile; - p->nMmap++; - *pp = (void*)(p->aData + iOfst); + if( iOfst+iAmt>p->sz ){ + *pp = 0; + }else{ + p->nMmap++; + *pp = (void*)(p->aData + iOfst); + } return SQLITE_OK; }@@ -46591,6 +47123,7 @@ p->mFlags = SQLITE_DESERIALIZE_RESIZEABLE | SQLITE_DESERIALIZE_FREEONCLOSE;
assert( pOutFlags!=0 ); /* True because flags==SQLITE_OPEN_MAIN_DB */ *pOutFlags = flags | SQLITE_OPEN_MEMORY; p->base.pMethods = &memdb_io_methods; + p->szMax = sqlite3GlobalConfig.mxMemdbSize; return SQLITE_OK; }@@ -46840,7 +47373,11 @@ rc = SQLITE_ERROR;
}else{ p->aData = pData; p->sz = szDb; + p->szAlloc = szBuf; p->szMax = szBuf; + if( p->szMax<sqlite3GlobalConfig.mxMemdbSize ){ + p->szMax = sqlite3GlobalConfig.mxMemdbSize; + } p->mFlags = mFlags; rc = SQLITE_OK; }@@ -47318,7 +47855,7 @@ **
** The PCache.pSynced variable is used to optimize searching for a dirty ** page to eject from the cache mid-transaction. It is better to eject ** a page that does not require a journal sync than one that does. -** Therefore, pSynced is maintained to that it *almost* always points +** Therefore, pSynced is maintained so that it *almost* always points ** to either the oldest page in the pDirty/pDirtyTail list that has a ** clear PGHDR_NEED_SYNC flag or to a page that is older than this one ** (so that the right page to eject can be found by following pDirtyPrev@@ -47529,9 +48066,10 @@ /* IMPLEMENTATION-OF: R-42059-47211 If the argument N is positive then the
** suggested cache size is set to N. */ return p->szCache; }else{ - /* IMPLEMENTATION-OF: R-61436-13639 If the argument N is negative, then - ** the number of cache pages is adjusted to use approximately abs(N*1024) - ** bytes of memory. */ + /* IMPLEMANTATION-OF: R-59858-46238 If the argument N is negative, then the + ** number of cache pages is adjusted to be a number of pages that would + ** use approximately abs(N*1024) bytes of memory based on the current + ** page size. */ return (int)((-1024*(i64)p->szCache)/(p->szPage+p->szExtra)); } }@@ -48142,6 +48680,15 @@ for(pDirty=pCache->pDirty; pDirty; pDirty=pDirty->pDirtyNext) nDirty++;
return nCache ? (int)(((i64)nDirty * 100) / nCache) : 0; } +#ifdef SQLITE_DIRECT_OVERFLOW_READ +/* +** Return true if there are one or more dirty pages in the cache. Else false. +*/ +SQLITE_PRIVATE int sqlite3PCacheIsDirty(PCache *pCache){ + return (pCache->pDirty!=0); +} +#endif + #if defined(SQLITE_CHECK_PAGES) || defined(SQLITE_DEBUG) /* ** For all dirty pages currently in the cache, invoke the specified@@ -48252,20 +48799,32 @@ ** Each cache entry is represented by an instance of the following
** structure. Unless SQLITE_PCACHE_SEPARATE_HEADER is defined, a buffer of ** PgHdr1.pCache->szPage bytes is allocated directly before this structure ** in memory. +** +** Note: Variables isBulkLocal and isAnchor were once type "u8". That works, +** but causes a 2-byte gap in the structure for most architectures (since +** pointers must be either 4 or 8-byte aligned). As this structure is located +** in memory directly after the associated page data, if the database is +** corrupt, code at the b-tree layer may overread the page buffer and +** read part of this structure before the corruption is detected. This +** can cause a valgrind error if the unitialized gap is accessed. Using u16 +** ensures there is no such gap, and therefore no bytes of unitialized memory +** in the structure. */ struct PgHdr1 { sqlite3_pcache_page page; /* Base class. Must be first. pBuf & pExtra */ unsigned int iKey; /* Key value (page number) */ - u8 isBulkLocal; /* This page from bulk local storage */ - u8 isAnchor; /* This is the PGroup.lru element */ + u16 isBulkLocal; /* This page from bulk local storage */ + u16 isAnchor; /* This is the PGroup.lru element */ PgHdr1 *pNext; /* Next in hash table chain */ PCache1 *pCache; /* Cache that currently owns this page */ PgHdr1 *pLruNext; /* Next in LRU list of unpinned pages */ PgHdr1 *pLruPrev; /* Previous in LRU list of unpinned pages */ + /* NB: pLruPrev is only valid if pLruNext!=0 */ }; /* -** A page is pinned if it is no on the LRU list +** A page is pinned if it is not on the LRU list. To be "pinned" means +** that the page is in active use and must not be deallocated. */ #define PAGE_IS_PINNED(p) ((p)->pLruNext==0) #define PAGE_IS_UNPINNED(p) ((p)->pLruNext!=0)@@ -48326,6 +48885,7 @@ unsigned int nMin; /* Minimum number of pages reserved */
unsigned int nMax; /* Configured "cache_size" value */ unsigned int n90pct; /* nMax*9/10 */ unsigned int iMaxKey; /* Largest key seen since xTruncate() */ + unsigned int nPurgeableDummy; /* pnPurgeable points here when not used*/ /* Hash table of all pages. The following variables may only be accessed ** when the accessor is holding the PGroup mutex.@@ -48460,6 +49020,7 @@ pX->page.pExtra = &pX[1];
pX->isBulkLocal = 1; pX->isAnchor = 0; pX->pNext = pCache->pFree; + pX->pLruPrev = 0; /* Initializing this saves a valgrind error */ pCache->pFree = pX; zBulk += pCache->szAlloc; }while( --nBulk );@@ -48635,6 +49196,7 @@ ** using sqlite3_config(SQLITE_CONFIG_PAGECACHE) option. If no such buffer
** exists, this function falls back to sqlite3Malloc(). */ SQLITE_PRIVATE void *sqlite3PageMalloc(int sz){ + assert( sz<=65536+8 ); /* These allocations are never very large */ return pcache1Alloc(sz); }@@ -48729,7 +49291,8 @@ assert( sqlite3_mutex_held(pPage->pCache->pGroup->mutex) );
pPage->pLruPrev->pLruNext = pPage->pLruNext; pPage->pLruNext->pLruPrev = pPage->pLruPrev; pPage->pLruNext = 0; - pPage->pLruPrev = 0; + /* pPage->pLruPrev = 0; + ** No need to clear pLruPrev as it is never accessed if pLruNext is 0 */ assert( pPage->isAnchor==0 ); assert( pPage->pCache->pGroup->lru.isAnchor==1 ); pPage->pCache->nRecyclable--;@@ -48922,6 +49485,7 @@ pGroup->mxPinned = 10;
}else{ pGroup = &pcache1.grp; } + pcache1EnterMutex(pGroup); if( pGroup->lru.isAnchor==0 ){ pGroup->lru.isAnchor = 1; pGroup->lru.pLruPrev = pGroup->lru.pLruNext = &pGroup->lru;@@ -48931,7 +49495,6 @@ pCache->szPage = szPage;
pCache->szExtra = szExtra; pCache->szAlloc = szPage + szExtra + ROUND8(sizeof(PgHdr1)); pCache->bPurgeable = (bPurgeable ? 1 : 0); - pcache1EnterMutex(pGroup); pcache1ResizeHash(pCache); if( bPurgeable ){ pCache->nMin = 10;@@ -48939,8 +49502,7 @@ pGroup->nMinPage += pCache->nMin;
pGroup->mxPinned = pGroup->nMaxPage + 10 - pGroup->nMinPage; pCache->pnPurgeable = &pGroup->nPurgeable; }else{ - static unsigned int dummyCurrentPage; - pCache->pnPurgeable = &dummyCurrentPage; + pCache->pnPurgeable = &pCache->nPurgeableDummy; } pcache1LeaveMutex(pGroup); if( pCache->nHash==0 ){@@ -49067,8 +49629,9 @@ pCache->nPage++;
pPage->iKey = iKey; pPage->pNext = pCache->apHash[h]; pPage->pCache = pCache; - pPage->pLruPrev = 0; pPage->pLruNext = 0; + /* pPage->pLruPrev = 0; + ** No need to clear pLruPrev since it is not accessed when pLruNext==0 */ *(void **)pPage->page.pExtra = 0; pCache->apHash[h] = pPage; if( iKey>pCache->iMaxKey ){@@ -49228,7 +49791,7 @@
/* It is an error to call this function if the page is already ** part of the PGroup LRU list. */ - assert( pPage->pLruPrev==0 && pPage->pLruNext==0 ); + assert( pPage->pLruNext==0 ); assert( PAGE_IS_PINNED(pPage) ); if( reuseUnlikely || pGroup->nPurgeable>pGroup->nMaxPage ){@@ -50905,19 +51468,33 @@ ** if( pPager->jfd->pMethods ){ ...
*/ #define isOpen(pFd) ((pFd)->pMethods!=0) +#ifdef SQLITE_DIRECT_OVERFLOW_READ /* -** Return true if this pager uses a write-ahead log to read page pgno. -** Return false if the pager reads pgno directly from the database. +** Return true if page pgno can be read directly from the database file +** by the b-tree layer. This is the case if: +** +** * the database file is open, +** * there are no dirty pages in the cache, and +** * the desired page is not currently in the wal file. */ -#if !defined(SQLITE_OMIT_WAL) && defined(SQLITE_DIRECT_OVERFLOW_READ) -SQLITE_PRIVATE int sqlite3PagerUseWal(Pager *pPager, Pgno pgno){ - u32 iRead = 0; - int rc; - if( pPager->pWal==0 ) return 0; - rc = sqlite3WalFindFrame(pPager->pWal, pgno, &iRead); - return rc || iRead; +SQLITE_PRIVATE int sqlite3PagerDirectReadOk(Pager *pPager, Pgno pgno){ + if( pPager->fd->pMethods==0 ) return 0; + if( sqlite3PCacheIsDirty(pPager->pPCache) ) return 0; +#ifdef SQLITE_HAS_CODEC + if( pPager->xCodec!=0 ) return 0; +#endif +#ifndef SQLITE_OMIT_WAL + if( pPager->pWal ){ + u32 iRead = 0; + int rc; + rc = sqlite3WalFindFrame(pPager->pWal, pgno, &iRead); + return (rc==SQLITE_OK && iRead==0); + } +#endif + return 1; } #endif + #ifndef SQLITE_OMIT_WAL # define pagerUseWal(x) ((x)->pWal!=0) #else@@ -53855,8 +54432,14 @@ if( pPager->eState>PAGER_OPEN && isOpen(pPager->fd) ){
rc = sqlite3OsFileSize(pPager->fd, &nByte); } if( rc==SQLITE_OK ){ - pNew = (char *)sqlite3PageMalloc(pageSize); - if( !pNew ) rc = SQLITE_NOMEM_BKPT; + /* 8 bytes of zeroed overrun space is sufficient so that the b-tree + * cell header parser will never run off the end of the allocation */ + pNew = (char *)sqlite3PageMalloc(pageSize+8); + if( !pNew ){ + rc = SQLITE_NOMEM_BKPT; + }else{ + memset(pNew+pageSize, 0, 8); + } } if( rc==SQLITE_OK ){@@ -53908,7 +54491,10 @@ if( mxPage>0 ){
pPager->mxPgno = mxPage; } assert( pPager->eState!=PAGER_OPEN ); /* Called only by OP_MaxPgcnt */ - assert( pPager->mxPgno>=pPager->dbSize ); /* OP_MaxPgcnt enforces this */ + /* assert( pPager->mxPgno>=pPager->dbSize ); */ + /* OP_MaxPgcnt ensures that the parameter passed to this function is not + ** less than the total number of valid pages in the database. But this + ** may be less than Pager.dbSize, and so the assert() above is not valid */ return pPager->mxPgno; }@@ -57101,7 +57687,11 @@ void (*xCodecSizeChng)(void*,int,int),
void (*xCodecFree)(void*), void *pCodec ){ - if( pPager->xCodecFree ) pPager->xCodecFree(pPager->pCodec); + if( pPager->xCodecFree ){ + pPager->xCodecFree(pPager->pCodec); + }else{ + pager_reset(pPager); + } pPager->xCodec = pPager->memDb ? 0 : xCodec; pPager->xCodecSizeChng = xCodecSizeChng; pPager->xCodecFree = xCodecFree;@@ -57230,8 +57820,12 @@ ** for the page moved there.
*/ pPg->flags &= ~PGHDR_NEED_SYNC; pPgOld = sqlite3PagerLookup(pPager, pgno); - assert( !pPgOld || pPgOld->nRef==1 ); + assert( !pPgOld || pPgOld->nRef==1 || CORRUPT_DB ); if( pPgOld ){ + if( pPgOld->nRef>1 ){ + sqlite3PagerUnrefNotNull(pPgOld); + return SQLITE_CORRUPT_BKPT; + } pPg->flags |= (pPgOld->flags&PGHDR_NEED_SYNC); if( pPager->tempFile ){ /* Do not discard pages from an in-memory database since we might@@ -57759,7 +58353,7 @@ ** sqlite3PagerSnapshotCheck().
*/ SQLITE_PRIVATE void sqlite3PagerSnapshotUnlock(Pager *pPager){ assert( pPager->pWal ); - return sqlite3WalSnapshotUnlock(pPager->pWal); + sqlite3WalSnapshotUnlock(pPager->pWal); } #endif /* SQLITE_ENABLE_SNAPSHOT */@@ -58360,7 +58954,7 @@ int rc = SQLITE_OK;
/* Enlarge the pWal->apWiData[] array if required */ if( pWal->nWiData<=iPage ){ - int nByte = sizeof(u32*)*(iPage+1); + sqlite3_int64 nByte = sizeof(u32*)*(iPage+1); volatile u32 **apNew; apNew = (volatile u32 **)sqlite3_realloc64((void *)pWal->apWiData, nByte); if( !apNew ){@@ -58464,6 +59058,7 @@ }
assert( nByte>=8 ); assert( (nByte&0x00000007)==0 ); + assert( nByte<=65536 ); if( nativeCksum ){ do {@@ -58771,6 +59366,7 @@ WalHashLoc sLoc; /* Hash table location */
int iLimit = 0; /* Zero values greater than this */ int nByte; /* Number of bytes to zero in aPgno[] */ int i; /* Used to iterate through aHash[] */ + int rc; /* Return code form walHashGet() */ assert( pWal->writeLock ); testcase( pWal->hdr.mxFrame==HASHTABLE_NPAGE_ONE-1 );@@ -58781,11 +59377,12 @@ if( pWal->hdr.mxFrame==0 ) return;
/* Obtain pointers to the hash-table and page-number array containing ** the entry that corresponds to frame pWal->hdr.mxFrame. It is guaranteed - ** that the page said hash-table and array reside on is already mapped. + ** that the page said hash-table and array reside on is already mapped.(1) */ assert( pWal->nWiData>walFramePage(pWal->hdr.mxFrame) ); assert( pWal->apWiData[walFramePage(pWal->hdr.mxFrame)] ); - walHashGet(pWal, walFramePage(pWal->hdr.mxFrame), &sLoc); + rc = walHashGet(pWal, walFramePage(pWal->hdr.mxFrame), &sLoc); + if( NEVER(rc) ) return; /* Defense-in-depth, in case (1) above is wrong */ /* Zero all hash-table entries that correspond to frame numbers greater ** than pWal->hdr.mxFrame.@@ -59399,7 +59996,7 @@ static int walIteratorInit(Wal *pWal, u32 nBackfill, WalIterator **pp){
WalIterator *p; /* Return value */ int nSegment; /* Number of segments to merge */ u32 iLast; /* Last frame in log */ - int nByte; /* Number of bytes to allocate */ + sqlite3_int64 nByte; /* Number of bytes to allocate */ int i; /* Iterator variable */ ht_slot *aTmp; /* Temp space used by merge-sort */ int rc = SQLITE_OK; /* Return Code */@@ -60690,9 +61287,9 @@ return rc;
} nCollide = HASHTABLE_NSLOT; for(iKey=walHash(pgno); sLoc.aHash[iKey]; iKey=walNextHash(iKey)){ - u32 iFrame = sLoc.aHash[iKey] + sLoc.iZero; - if( iFrame<=iLast && iFrame>=pWal->minFrame - && sLoc.aPgno[sLoc.aHash[iKey]]==pgno ){ + u32 iH = sLoc.aHash[iKey]; + u32 iFrame = iH + sLoc.iZero; + if( iFrame<=iLast && iFrame>=pWal->minFrame && sLoc.aPgno[iH]==pgno ){ assert( iFrame>iRead || CORRUPT_DB ); iRead = iFrame; }@@ -61935,7 +62532,7 @@ u8 nOverflow; /* Number of overflow cell bodies in aCell[] */
u16 maxLocal; /* Copy of BtShared.maxLocal or BtShared.maxLeaf */ u16 minLocal; /* Copy of BtShared.minLocal or BtShared.minLeaf */ u16 cellOffset; /* Index in aData of first cell pointer */ - u16 nFree; /* Number of free bytes on the page */ + int nFree; /* Number of free bytes on the page. -1 for unknown */ u16 nCell; /* Number of cells on this page, local and ovfl */ u16 maskPage; /* Mask for page offset */ u16 aiOvfl[4]; /* Insert the i-th overflow cell before the aiOvfl-th@@ -62143,9 +62740,16 @@ ** Fields in this structure are accessed under the BtShared.mutex
** found at self->pBt->mutex. ** ** skipNext meaning: -** eState==SKIPNEXT && skipNext>0: Next sqlite3BtreeNext() is no-op. -** eState==SKIPNEXT && skipNext<0: Next sqlite3BtreePrevious() is no-op. -** eState==FAULT: Cursor fault with skipNext as error code. +** The meaning of skipNext depends on the value of eState: +** +** eState Meaning of skipNext +** VALID skipNext is meaningless and is ignored +** INVALID skipNext is meaningless and is ignored +** SKIPNEXT sqlite3BtreeNext() is a no-op if skipNext>0 and +** sqlite3BtreePrevious() is no-op if skipNext<0. +** REQUIRESEEK restoreCursorPosition() restores the cursor to +** eState=SKIPNEXT if skipNext!=0 +** FAULT skipNext holds the cursor fault error code. */ struct BtCursor { u8 eState; /* One of the CURSOR_XXX constants (see below) */@@ -63309,13 +63913,19 @@ if( pCur->curIntKey ){
/* Only the rowid is required for a table btree */ pCur->nKey = sqlite3BtreeIntegerKey(pCur); }else{ - /* For an index btree, save the complete key content */ + /* For an index btree, save the complete key content. It is possible + ** that the current key is corrupt. In that case, it is possible that + ** the sqlite3VdbeRecordUnpack() function may overread the buffer by + ** up to the size of 1 varint plus 1 8-byte value when the cursor + ** position is restored. Hence the 17 bytes of padding allocated + ** below. */ void *pKey; pCur->nKey = sqlite3BtreePayloadSize(pCur); - pKey = sqlite3Malloc( pCur->nKey ); + pKey = sqlite3Malloc( pCur->nKey + 9 + 8 ); if( pKey ){ rc = sqlite3BtreePayload(pCur, 0, (int)pCur->nKey, pKey); if( rc==SQLITE_OK ){ + memset(((u8*)pKey)+pCur->nKey, 0, 9+8); pCur->pKey = pKey; }else{ sqlite3_free(pKey);@@ -63447,11 +64057,12 @@ int rc; /* Status code */
UnpackedRecord *pIdxKey; /* Unpacked index key */ if( pKey ){ + KeyInfo *pKeyInfo = pCur->pKeyInfo; assert( nKey==(i64)(int)nKey ); - pIdxKey = sqlite3VdbeAllocUnpackedRecord(pCur->pKeyInfo); + pIdxKey = sqlite3VdbeAllocUnpackedRecord(pKeyInfo); if( pIdxKey==0 ) return SQLITE_NOMEM_BKPT; - sqlite3VdbeRecordUnpack(pCur->pKeyInfo, (int)nKey, pKey, pIdxKey); - if( pIdxKey->nField==0 ){ + sqlite3VdbeRecordUnpack(pKeyInfo, (int)nKey, pKey, pIdxKey); + if( pIdxKey->nField==0 || pIdxKey->nField>pKeyInfo->nAllField ){ rc = SQLITE_CORRUPT_BKPT; goto moveto_done; }@@ -63475,19 +64086,23 @@ ** saveCursorPosition().
*/ static int btreeRestoreCursorPosition(BtCursor *pCur){ int rc; - int skipNext; + int skipNext = 0; assert( cursorOwnsBtShared(pCur) ); assert( pCur->eState>=CURSOR_REQUIRESEEK ); if( pCur->eState==CURSOR_FAULT ){ return pCur->skipNext; } pCur->eState = CURSOR_INVALID; - rc = btreeMoveto(pCur, pCur->pKey, pCur->nKey, 0, &skipNext); + if( sqlite3FaultSim(410) ){ + rc = SQLITE_IOERR; + }else{ + rc = btreeMoveto(pCur, pCur->pKey, pCur->nKey, 0, &skipNext); + } if( rc==SQLITE_OK ){ sqlite3_free(pCur->pKey); pCur->pKey = 0; assert( pCur->eState==CURSOR_VALID || pCur->eState==CURSOR_INVALID ); - pCur->skipNext |= skipNext; + if( skipNext ) pCur->skipNext = skipNext; if( pCur->skipNext && pCur->eState==CURSOR_VALID ){ pCur->eState = CURSOR_SKIPNEXT; }@@ -63557,7 +64172,6 @@ }
if( pCur->eState!=CURSOR_VALID ){ *pDifferentRow = 1; }else{ - assert( pCur->skipNext==0 ); *pDifferentRow = 0; } return SQLITE_OK;@@ -63641,6 +64255,13 @@ if( rc!=SQLITE_OK ){
*pRC = rc; return; } + if( ((char*)sqlite3PagerGetExtra(pDbPage))[0]!=0 ){ + /* The first byte of the extra data is the MemPage.isInit byte. + ** If that byte is set, it means this page is also being used + ** as a btree page. */ + *pRC = SQLITE_CORRUPT_BKPT; + goto ptrmap_exit; + } offset = PTRMAP_PTROFFSET(iPtrmap, key); if( offset<0 ){ *pRC = SQLITE_CORRUPT_BKPT;@@ -63703,7 +64324,7 @@
#else /* if defined SQLITE_OMIT_AUTOVACUUM */ #define ptrmapPut(w,x,y,z,rc) #define ptrmapGet(w,x,y,z) SQLITE_OK - #define ptrmapPutOvflPtr(x, y, rc) + #define ptrmapPutOvflPtr(x, y, z, rc) #endif /*@@ -63996,17 +64617,24 @@ #endif
#ifndef SQLITE_OMIT_AUTOVACUUM /* -** If the cell pCell, part of page pPage contains a pointer -** to an overflow page, insert an entry into the pointer-map -** for the overflow page. +** The cell pCell is currently part of page pSrc but will ultimately be part +** of pPage. (pSrc and pPager are often the same.) If pCell contains a +** pointer to an overflow page, insert an entry into the pointer-map for +** the overflow page that will be valid after pCell has been moved to pPage. */ -static void ptrmapPutOvflPtr(MemPage *pPage, u8 *pCell, int *pRC){ +static void ptrmapPutOvflPtr(MemPage *pPage, MemPage *pSrc, u8 *pCell,int *pRC){ CellInfo info; if( *pRC ) return; assert( pCell!=0 ); pPage->xParseCell(pPage, pCell, &info); if( info.nLocal<info.nPayload ){ - Pgno ovfl = get4byte(&pCell[info.nSize-4]); + Pgno ovfl; + if( SQLITE_WITHIN(pSrc->aDataEnd, pCell, pCell+info.nLocal) ){ + testcase( pSrc!=pPage ); + *pRC = SQLITE_CORRUPT_BKPT; + return; + } + ovfl = get4byte(&pCell[info.nSize-4]); ptrmapPut(pPage->pBt, ovfl, PTRMAP_OVERFLOW1, pPage->pgno, pRC); } }@@ -64050,7 +64678,7 @@ src = data = pPage->aData;
hdr = pPage->hdrOffset; cellOffset = pPage->cellOffset; nCell = pPage->nCell; - assert( nCell==get2byte(&data[hdr+3]) ); + assert( nCell==get2byte(&data[hdr+3]) || CORRUPT_DB ); iCellFirst = cellOffset + 2*nCell; usableSize = pPage->pBt->usableSize;@@ -64061,19 +64689,10 @@ ** offsets to each pointer in the cell-pointer array than it is to
** reconstruct the entire page. */ if( (int)data[hdr+7]<=nMaxFrag ){ int iFree = get2byte(&data[hdr+1]); + if( iFree>usableSize-4 ) return SQLITE_CORRUPT_PAGE(pPage); if( iFree ){ int iFree2 = get2byte(&data[iFree]); - - /* pageFindSlot() has already verified that free blocks are sorted - ** in order of offset within the page, and that no block extends - ** past the end of the page. Provided the two free slots do not - ** overlap, this guarantees that the memmove() calls below will not - ** overwrite the usableSize byte buffer, even if the database page - ** is corrupt. */ - assert( iFree2==0 || iFree2>iFree ); - assert( iFree+get2byte(&data[iFree+2]) <= usableSize ); - assert( iFree2==0 || iFree2+get2byte(&data[iFree2+2]) <= usableSize ); - + if( iFree2>usableSize-4 ) return SQLITE_CORRUPT_PAGE(pPage); if( 0==iFree2 || (data[iFree2]==0 && data[iFree2+1]==0) ){ u8 *pEnd = &data[cellOffset + nCell*2]; u8 *pAddr;@@ -64084,12 +64703,15 @@ if( top>=iFree ){
return SQLITE_CORRUPT_PAGE(pPage); } if( iFree2 ){ - assert( iFree+sz<=iFree2 ); /* Verified by pageFindSlot() */ + if( iFree+sz>iFree2 ) return SQLITE_CORRUPT_PAGE(pPage); sz2 = get2byte(&data[iFree2+2]); - assert( iFree+sz+sz2+iFree2-(iFree+sz) <= usableSize ); + if( iFree2+sz2 > usableSize ) return SQLITE_CORRUPT_PAGE(pPage); memmove(&data[iFree+sz+sz2], &data[iFree+sz], iFree2-(iFree+sz)); sz += sz2; + }else if( iFree+sz>usableSize ){ + return SQLITE_CORRUPT_PAGE(pPage); } + cbrk = top+sz; assert( cbrk+(iFree-top) <= usableSize ); memmove(&data[cbrk], &data[top], iFree-top);@@ -64140,6 +64762,7 @@ }
data[hdr+7] = 0; defragment_out: + assert( pPage->nFree>=0 ); if( data[hdr+7]+cbrk-iCellFirst!=pPage->nFree ){ return SQLITE_CORRUPT_PAGE(pPage); }@@ -64167,16 +64790,16 @@ ** will be ignored if adding the extra space to the fragmentation count
** causes the fragmentation count to exceed 60. */ static u8 *pageFindSlot(MemPage *pPg, int nByte, int *pRc){ - const int hdr = pPg->hdrOffset; - u8 * const aData = pPg->aData; - int iAddr = hdr + 1; - int pc = get2byte(&aData[iAddr]); - int x; - int usableSize = pPg->pBt->usableSize; - int size; /* Size of the free slot */ + const int hdr = pPg->hdrOffset; /* Offset to page header */ + u8 * const aData = pPg->aData; /* Page data */ + int iAddr = hdr + 1; /* Address of ptr to pc */ + int pc = get2byte(&aData[iAddr]); /* Address of a free slot */ + int x; /* Excess size of the slot */ + int maxPC = pPg->pBt->usableSize - nByte; /* Max address for a usable slot */ + int size; /* Size of the free slot */ assert( pc>0 ); - while( pc<=usableSize-4 ){ + while( pc<=maxPC ){ /* EVIDENCE-OF: R-22710-53328 The third and fourth bytes of each ** freeblock form a big-endian integer which is the size of the freeblock ** in bytes, including the 4-byte header. */@@ -64184,10 +64807,7 @@ size = get2byte(&aData[pc+2]);
if( (x = size - nByte)>=0 ){ testcase( x==4 ); testcase( x==3 ); - if( size+pc > usableSize ){ - *pRc = SQLITE_CORRUPT_PAGE(pPg); - return 0; - }else if( x<4 ){ + if( x<4 ){ /* EVIDENCE-OF: R-11498-58022 In a well-formed b-tree page, the total ** number of bytes in fragments may not exceed 60. */ if( aData[hdr+7]>57 ) return 0;@@ -64196,21 +64816,31 @@ /* Remove the slot from the free-list. Update the number of
** fragmented bytes within the page. */ memcpy(&aData[iAddr], &aData[pc], 2); aData[hdr+7] += (u8)x; + }else if( x+pc > maxPC ){ + /* This slot extends off the end of the usable part of the page */ + *pRc = SQLITE_CORRUPT_PAGE(pPg); + return 0; }else{ /* The slot remains on the free-list. Reduce its size to account - ** for the portion used by the new allocation. */ + ** for the portion used by the new allocation. */ put2byte(&aData[pc+2], x); } return &aData[pc + x]; } iAddr = pc; pc = get2byte(&aData[pc]); - if( pc<iAddr+size ) break; + if( pc<=iAddr+size ){ + if( pc ){ + /* The next slot in the chain is not past the end of the current slot */ + *pRc = SQLITE_CORRUPT_PAGE(pPg); + } + return 0; + } } - if( pc ){ + if( pc>maxPC+nByte-4 ){ + /* The free slot chain extends off the end of the page */ *pRc = SQLITE_CORRUPT_PAGE(pPg); } - return 0; }@@ -64251,7 +64881,7 @@ ** then the cell content offset of an empty page wants to be 65536.
** However, that integer is too large to be stored in a 2-byte unsigned ** integer, so a value of 0 is used in its place. */ top = get2byte(&data[hdr+5]); - assert( top<=(int)pPage->pBt->usableSize ); /* Prevent by getAndInitPage() */ + assert( top<=(int)pPage->pBt->usableSize ); /* by btreeComputeFreeSpace() */ if( gap>top ){ if( top==0 && pPage->pBt->usableSize==65536 ){ top = 65536;@@ -64260,9 +64890,9 @@ return SQLITE_CORRUPT_PAGE(pPage);
} } - /* If there is enough space between gap and top for one more cell pointer - ** array entry offset, and if the freelist is not empty, then search the - ** freelist looking for a free slot big enough to satisfy the request. + /* If there is enough space between gap and top for one more cell pointer, + ** and if the freelist is not empty, then search the + ** freelist looking for a slot big enough to satisfy the request. */ testcase( gap+2==top ); testcase( gap+1==top );@@ -64284,6 +64914,7 @@ */
testcase( gap+2+nByte==top ); if( gap+2+nByte>top ){ assert( pPage->nCell>0 || CORRUPT_DB ); + assert( pPage->nFree>=0 ); rc = defragmentPage(pPage, MIN(4, pPage->nFree - (2+nByte))); if( rc ) return rc; top = get2byteNotZero(&data[hdr+5]);@@ -64292,7 +64923,7 @@ }
/* Allocate memory from the gap in between the cell pointer array - ** and the cell content area. The btreeInitPage() call has already + ** and the cell content area. The btreeComputeFreeSpace() call has already ** validated the freelist. Given that the freelist is valid, there ** is no way that the allocation can extend off the end of the page. ** The assert() below verifies the previous sentence.@@ -64311,7 +64942,7 @@ ** and the size of the block is iSize bytes.
** ** Adjacent freeblocks are coalesced. ** -** Note that even though the freeblock list was checked by btreeInitPage(), +** Even though the freeblock list was checked by btreeComputeFreeSpace(), ** that routine will not detect overlap between cells or freeblocks. Nor ** does it detect cells or freeblocks that encrouch into the reserved bytes ** at the end of the page. So do additional corruption checks inside this@@ -64473,21 +65104,14 @@ return SQLITE_OK;
} /* -** Initialize the auxiliary information for a disk block. -** -** Return SQLITE_OK on success. If we see that the page does -** not contain a well-formed database page, then return -** SQLITE_CORRUPT. Note that a return of SQLITE_OK does not -** guarantee that the page is well-formed. It only shows that -** we failed to detect any corruption. +** Compute the amount of freespace on the page. In other words, fill +** in the pPage->nFree field. */ -static int btreeInitPage(MemPage *pPage){ +static int btreeComputeFreeSpace(MemPage *pPage){ int pc; /* Address of a freeblock within pPage->aData[] */ u8 hdr; /* Offset to beginning of page header */ u8 *data; /* Equal to pPage->aData */ - BtShared *pBt; /* The main btree structure */ int usableSize; /* Amount of usable space on each page */ - u16 cellOffset; /* Offset from start of page to first cell pointer */ int nFree; /* Number of unused bytes on the page */ int top; /* First byte of the cell content area */ int iCellFirst; /* First allowable cell or freeblock offset */@@ -64499,71 +65123,18 @@ assert( sqlite3_mutex_held(pPage->pBt->mutex) );
assert( pPage->pgno==sqlite3PagerPagenumber(pPage->pDbPage) ); assert( pPage == sqlite3PagerGetExtra(pPage->pDbPage) ); assert( pPage->aData == sqlite3PagerGetData(pPage->pDbPage) ); - assert( pPage->isInit==0 ); + assert( pPage->isInit==1 ); + assert( pPage->nFree<0 ); - pBt = pPage->pBt; + usableSize = pPage->pBt->usableSize; hdr = pPage->hdrOffset; data = pPage->aData; - /* EVIDENCE-OF: R-28594-02890 The one-byte flag at offset 0 indicating - ** the b-tree page type. */ - if( decodeFlags(pPage, data[hdr]) ){ - return SQLITE_CORRUPT_PAGE(pPage); - } - assert( pBt->pageSize>=512 && pBt->pageSize<=65536 ); - pPage->maskPage = (u16)(pBt->pageSize - 1); - pPage->nOverflow = 0; - usableSize = pBt->usableSize; - pPage->cellOffset = cellOffset = hdr + 8 + pPage->childPtrSize; - pPage->aDataEnd = &data[usableSize]; - pPage->aCellIdx = &data[cellOffset]; - pPage->aDataOfst = &data[pPage->childPtrSize]; /* EVIDENCE-OF: R-58015-48175 The two-byte integer at offset 5 designates ** the start of the cell content area. A zero value for this integer is ** interpreted as 65536. */ top = get2byteNotZero(&data[hdr+5]); - /* EVIDENCE-OF: R-37002-32774 The two-byte integer at offset 3 gives the - ** number of cells on the page. */ - pPage->nCell = get2byte(&data[hdr+3]); - if( pPage->nCell>MX_CELL(pBt) ){ - /* To many cells for a single page. The page must be corrupt */ - return SQLITE_CORRUPT_PAGE(pPage); - } - testcase( pPage->nCell==MX_CELL(pBt) ); - /* EVIDENCE-OF: R-24089-57979 If a page contains no cells (which is only - ** possible for a root page of a table that contains no rows) then the - ** offset to the cell content area will equal the page size minus the - ** bytes of reserved space. */ - assert( pPage->nCell>0 || top==usableSize || CORRUPT_DB ); - - /* A malformed database page might cause us to read past the end - ** of page when parsing a cell. - ** - ** The following block of code checks early to see if a cell extends - ** past the end of a page boundary and causes SQLITE_CORRUPT to be - ** returned if it does. - */ - iCellFirst = cellOffset + 2*pPage->nCell; + iCellFirst = hdr + 8 + pPage->childPtrSize + 2*pPage->nCell; iCellLast = usableSize - 4; - if( pBt->db->flags & SQLITE_CellSizeCk ){ - int i; /* Index into the cell pointer array */ - int sz; /* Size of a cell */ - - if( !pPage->leaf ) iCellLast--; - for(i=0; i<pPage->nCell; i++){ - pc = get2byteAligned(&data[cellOffset+i*2]); - testcase( pc==iCellFirst ); - testcase( pc==iCellLast ); - if( pc<iCellFirst || pc>iCellLast ){ - return SQLITE_CORRUPT_PAGE(pPage); - } - sz = pPage->xCellSize(pPage, &data[pc]); - testcase( pc+sz==usableSize ); - if( pc+sz>usableSize ){ - return SQLITE_CORRUPT_PAGE(pPage); - } - } - if( !pPage->leaf ) iCellLast++; - } /* Compute the total free space on the page ** EVIDENCE-OF: R-23588-34450 The two-byte integer at offset 1 gives the@@ -64607,11 +65178,104 @@ ** of the page, then the page must be corrupted. This check also
** serves to verify that the offset to the start of the cell-content ** area, according to the page header, lies within the page. */ - if( nFree>usableSize ){ + if( nFree>usableSize || nFree<iCellFirst ){ return SQLITE_CORRUPT_PAGE(pPage); } pPage->nFree = (u16)(nFree - iCellFirst); + return SQLITE_OK; +} + +/* +** Do additional sanity check after btreeInitPage() if +** PRAGMA cell_size_check=ON +*/ +static SQLITE_NOINLINE int btreeCellSizeCheck(MemPage *pPage){ + int iCellFirst; /* First allowable cell or freeblock offset */ + int iCellLast; /* Last possible cell or freeblock offset */ + int i; /* Index into the cell pointer array */ + int sz; /* Size of a cell */ + int pc; /* Address of a freeblock within pPage->aData[] */ + u8 *data; /* Equal to pPage->aData */ + int usableSize; /* Maximum usable space on the page */ + int cellOffset; /* Start of cell content area */ + + iCellFirst = pPage->cellOffset + 2*pPage->nCell; + usableSize = pPage->pBt->usableSize; + iCellLast = usableSize - 4; + data = pPage->aData; + cellOffset = pPage->cellOffset; + if( !pPage->leaf ) iCellLast--; + for(i=0; i<pPage->nCell; i++){ + pc = get2byteAligned(&data[cellOffset+i*2]); + testcase( pc==iCellFirst ); + testcase( pc==iCellLast ); + if( pc<iCellFirst || pc>iCellLast ){ + return SQLITE_CORRUPT_PAGE(pPage); + } + sz = pPage->xCellSize(pPage, &data[pc]); + testcase( pc+sz==usableSize ); + if( pc+sz>usableSize ){ + return SQLITE_CORRUPT_PAGE(pPage); + } + } + return SQLITE_OK; +} + +/* +** Initialize the auxiliary information for a disk block. +** +** Return SQLITE_OK on success. If we see that the page does +** not contain a well-formed database page, then return +** SQLITE_CORRUPT. Note that a return of SQLITE_OK does not +** guarantee that the page is well-formed. It only shows that +** we failed to detect any corruption. +*/ +static int btreeInitPage(MemPage *pPage){ + u8 *data; /* Equal to pPage->aData */ + BtShared *pBt; /* The main btree structure */ + + assert( pPage->pBt!=0 ); + assert( pPage->pBt->db!=0 ); + assert( sqlite3_mutex_held(pPage->pBt->mutex) ); + assert( pPage->pgno==sqlite3PagerPagenumber(pPage->pDbPage) ); + assert( pPage == sqlite3PagerGetExtra(pPage->pDbPage) ); + assert( pPage->aData == sqlite3PagerGetData(pPage->pDbPage) ); + assert( pPage->isInit==0 ); + + pBt = pPage->pBt; + data = pPage->aData + pPage->hdrOffset; + /* EVIDENCE-OF: R-28594-02890 The one-byte flag at offset 0 indicating + ** the b-tree page type. */ + if( decodeFlags(pPage, data[0]) ){ + return SQLITE_CORRUPT_PAGE(pPage); + } + assert( pBt->pageSize>=512 && pBt->pageSize<=65536 ); + pPage->maskPage = (u16)(pBt->pageSize - 1); + pPage->nOverflow = 0; + pPage->cellOffset = pPage->hdrOffset + 8 + pPage->childPtrSize; + pPage->aCellIdx = data + pPage->childPtrSize + 8; + pPage->aDataEnd = pPage->aData + pBt->usableSize; + pPage->aDataOfst = pPage->aData + pPage->childPtrSize; + /* EVIDENCE-OF: R-37002-32774 The two-byte integer at offset 3 gives the + ** number of cells on the page. */ + pPage->nCell = get2byte(&data[3]); + if( pPage->nCell>MX_CELL(pBt) ){ + /* To many cells for a single page. The page must be corrupt */ + return SQLITE_CORRUPT_PAGE(pPage); + } + testcase( pPage->nCell==MX_CELL(pBt) ); + /* EVIDENCE-OF: R-24089-57979 If a page contains no cells (which is only + ** possible for a root page of a table that contains no rows) then the + ** offset to the cell content area will equal the page size minus the + ** bytes of reserved space. */ + assert( pPage->nCell>0 + || get2byteNotZero(&data[5])==(int)pBt->usableSize + || CORRUPT_DB ); + pPage->nFree = -1; /* Indicate that this value is yet uncomputed */ pPage->isInit = 1; + if( pBt->db->flags & SQLITE_CellSizeCk ){ + return btreeCellSizeCheck(pPage); + } return SQLITE_OK; }@@ -64754,19 +65418,18 @@ assert( pCur==0 || pCur->iPage>0 );
if( pgno>btreePagecount(pBt) ){ rc = SQLITE_CORRUPT_BKPT; - goto getAndInitPage_error; + goto getAndInitPage_error1; } rc = sqlite3PagerGet(pBt->pPager, pgno, (DbPage**)&pDbPage, bReadOnly); if( rc ){ - goto getAndInitPage_error; + goto getAndInitPage_error1; } *ppPage = (MemPage*)sqlite3PagerGetExtra(pDbPage); if( (*ppPage)->isInit==0 ){ btreePageFromDbPage(pDbPage, pgno, pBt); rc = btreeInitPage(*ppPage); if( rc!=SQLITE_OK ){ - releasePage(*ppPage); - goto getAndInitPage_error; + goto getAndInitPage_error2; } } assert( (*ppPage)->pgno==pgno );@@ -64776,12 +65439,13 @@ /* If obtaining a child page for a cursor, we must verify that the page is
** compatible with the root page. */ if( pCur && ((*ppPage)->nCell<1 || (*ppPage)->intKey!=pCur->curIntKey) ){ rc = SQLITE_CORRUPT_PGNO(pgno); - releasePage(*ppPage); - goto getAndInitPage_error; + goto getAndInitPage_error2; } return SQLITE_OK; -getAndInitPage_error: +getAndInitPage_error2: + releasePage(*ppPage); +getAndInitPage_error1: if( pCur ){ pCur->iPage--; pCur->pPage = pCur->apPage[pCur->iPage];@@ -65641,9 +66305,9 @@ */
static int lockBtree(BtShared *pBt){ int rc; /* Result code from subfunctions */ MemPage *pPage1; /* Page 1 of the database file */ - int nPage; /* Number of pages in the database */ - int nPageFile = 0; /* Number of pages in the database file */ - int nPageHeader; /* Number of pages in the database according to hdr */ + u32 nPage; /* Number of pages in the database */ + u32 nPageFile = 0; /* Number of pages in the database file */ + u32 nPageHeader; /* Number of pages in the database according to hdr */ assert( sqlite3_mutex_held(pBt->mutex) ); assert( pBt->pPage1==0 );@@ -65656,7 +66320,7 @@ /* Do some checking to help insure the file we opened really is
** a valid database file. */ nPage = nPageHeader = get4byte(28+(u8*)pPage1->aData); - sqlite3PagerPagecount(pBt->pPager, &nPageFile); + sqlite3PagerPagecount(pBt->pPager, (int*)&nPageFile); if( nPage==0 || memcmp(24+(u8*)pPage1->aData, 92+(u8*)pPage1->aData,4)!=0 ){ nPage = nPageFile; }@@ -65737,6 +66401,7 @@ || pageSize<=256
){ goto page1_init_failed; } + pBt->btsFlags |= BTS_PAGESIZE_FIXED; assert( (pageSize & 7)==0 ); /* EVIDENCE-OF: R-59310-51205 The "reserved space" size in the 1-byte ** integer at offset 20 is the number of bytes of space at the end of@@ -65761,7 +66426,7 @@ rc = sqlite3PagerSetPagesize(pBt->pPager, &pBt->pageSize,
pageSize-usableSize); return rc; } - if( (pBt->db->flags & SQLITE_WriteSchema)==0 && nPage>nPageFile ){ + if( sqlite3WritableSchema(pBt->db)==0 && nPage>nPageFile ){ rc = SQLITE_CORRUPT_BKPT; goto page1_init_failed; }@@ -66127,7 +66792,7 @@
for(i=0; i<nCell; i++){ u8 *pCell = findCell(pPage, i); - ptrmapPutOvflPtr(pPage, pCell, &rc); + ptrmapPutOvflPtr(pPage, pPage, pCell, &rc); if( !pPage->leaf ){ Pgno childPgno = get4byte(pCell);@@ -66235,6 +66900,7 @@ assert( eType==PTRMAP_OVERFLOW2 || eType==PTRMAP_OVERFLOW1 ||
eType==PTRMAP_BTREE || eType==PTRMAP_ROOTPAGE ); assert( sqlite3_mutex_held(pBt->mutex) ); assert( pDbPage->pBt==pBt ); + if( iDbPage<3 ) return SQLITE_CORRUPT_BKPT; /* Move page iDbPage from its current location to page number iFreePage */ TRACE(("AUTOVACUUM: Moving %d to free page %d (ptr page %d type %d)\n",@@ -66741,6 +67407,18 @@ return rc;
} /* +** Set the pBt->nPage field correctly, according to the current +** state of the database. Assume pBt->pPage1 is valid. +*/ +static void btreeSetNPage(BtShared *pBt, MemPage *pPage1){ + int nPage = get4byte(&pPage1->aData[28]); + testcase( nPage==0 ); + if( nPage==0 ) sqlite3PagerPagecount(pBt->pPager, &nPage); + testcase( pBt->nPage!=nPage ); + pBt->nPage = nPage; +} + +/* ** Rollback the transaction in progress. ** ** If tripCode is not SQLITE_OK then cursors will be invalidated (tripped).@@ -66785,11 +67463,7 @@ /* The rollback may have destroyed the pPage1->aData value. So
** call btreeGetPage() on page 1 again to make ** sure pPage1->aData is set correctly. */ if( btreeGetPage(pBt, 1, &pPage1, 0)==SQLITE_OK ){ - int nPage = get4byte(28+(u8*)pPage1->aData); - testcase( nPage==0 ); - if( nPage==0 ) sqlite3PagerPagecount(pBt->pPager, &nPage); - testcase( pBt->nPage!=nPage ); - pBt->nPage = nPage; + btreeSetNPage(pBt, pPage1); releasePageOne(pPage1); } assert( countValidCursors(pBt, 1)==0 );@@ -66869,12 +67543,11 @@ if( iSavepoint<0 && (pBt->btsFlags & BTS_INITIALLY_EMPTY)!=0 ){
pBt->nPage = 0; } rc = newDatabase(pBt); - pBt->nPage = get4byte(28 + pBt->pPage1->aData); + btreeSetNPage(pBt, pBt->pPage1); - /* The database size was written into the offset 28 of the header - ** when the transaction started, so we know that the value at offset - ** 28 is nonzero. */ - assert( pBt->nPage>0 ); + /* pBt->nPage might be zero if the database was corrupt when + ** the transaction was started. Otherwise, it must be at least 1. */ + assert( CORRUPT_DB || pBt->nPage>0 ); } sqlite3BtreeLeave(p); }@@ -67052,6 +67725,7 @@ unlockBtreeIfUnused(pBt);
sqlite3_free(pCur->aOverflow); sqlite3_free(pCur->pKey); sqlite3BtreeLeave(pBtree); + pCur->pBtree = 0; } return SQLITE_OK; }@@ -67151,6 +67825,25 @@ return pCur->info.nPayload;
} /* +** Return an upper bound on the size of any record for the table +** that the cursor is pointing into. +** +** This is an optimization. Everything will still work if this +** routine always returns 2147483647 (which is the largest record +** that SQLite can handle) or more. But returning a smaller value might +** prevent large memory allocations when trying to interpret a +** corrupt datrabase. +** +** The current implementation merely returns the size of the underlying +** database file. +*/ +SQLITE_PRIVATE sqlite3_int64 sqlite3BtreeMaxRecordSize(BtCursor *pCur){ + assert( cursorHoldsMutex(pCur) ); + assert( pCur->eState==CURSOR_VALID ); + return pCur->pBt->pageSize * (sqlite3_int64)pCur->pBt->nPage; +} + +/* ** Given the page number of an overflow page in the database (parameter ** ovfl), this function finds the page number of the next page in the ** linked list of overflow pages. If possible, it uses the auto-vacuum@@ -67406,9 +68099,6 @@ }else{
/* Need to read this page properly. It contains some of the ** range of data that is being read (eOp==0) or written (eOp!=0). */ -#ifdef SQLITE_DIRECT_OVERFLOW_READ - sqlite3_file *fd; /* File from which to do direct overflow read */ -#endif int a = amt; if( a + offset > ovflSize ){ a = ovflSize - offset;@@ -67419,7 +68109,7 @@ /* If all the following are true:
** ** 1) this is a read operation, and ** 2) data is required from the start of this overflow page, and - ** 3) there is no open write-transaction, and + ** 3) there are no dirty pages in the page-cache ** 4) the database is file-backed, and ** 5) the page is not in the WAL file ** 6) at least 4 bytes have already been read into the output buffer@@ -67430,11 +68120,10 @@ ** up loading large records that span many overflow pages.
*/ if( eOp==0 /* (1) */ && offset==0 /* (2) */ - && pBt->inTransaction==TRANS_READ /* (3) */ - && (fd = sqlite3PagerFile(pBt->pPager))->pMethods /* (4) */ - && 0==sqlite3PagerUseWal(pBt->pPager, nextPage) /* (5) */ + && sqlite3PagerDirectReadOk(pBt->pPager, nextPage) /* (3,4,5) */ && &pBuf[-4]>=pBufStart /* (6) */ ){ + sqlite3_file *fd = sqlite3PagerFile(pBt->pPager); u8 aSave[4]; u8 *aWrite = &pBuf[-4]; assert( aWrite>=pBufStart ); /* due to (6) */@@ -67844,23 +68533,6 @@ }
return rc; } -/* -** This function is a no-op if cursor pCur does not point to a valid row. -** Otherwise, if pCur is valid, configure it so that the next call to -** sqlite3BtreeNext() is a no-op. -*/ -#ifndef SQLITE_OMIT_WINDOWFUNC -SQLITE_PRIVATE void sqlite3BtreeSkipNext(BtCursor *pCur){ - /* We believe that the cursor must always be in the valid state when - ** this routine is called, but the proof is difficult, so we add an - ** ALWaYS() test just in case we are wrong. */ - if( ALWAYS(pCur->eState==CURSOR_VALID) ){ - pCur->eState = CURSOR_SKIPNEXT; - pCur->skipNext = 1; - } -} -#endif /* SQLITE_OMIT_WINDOWFUNC */ - /* Move the cursor to the last entry in the table. Return SQLITE_OK ** on success. Set *pRes to 0 if the cursor actually points to something ** or set *pRes to 1 if the table is empty.@@ -67883,6 +68555,7 @@ }
assert( pCur->ix==pCur->pPage->nCell-1 ); assert( pCur->pPage->leaf ); #endif + *pRes = 0; return SQLITE_OK; }@@ -67968,7 +68641,7 @@ /* If the requested key is one more than the previous key, then
** try to get there using sqlite3BtreeNext() rather than a full ** binary search. This is an optimization only. The correct answer ** is still obtained without this case, only a little more slowely */ - if( pCur->info.nKey+1==intKey && !pCur->skipNext ){ + if( pCur->info.nKey+1==intKey ){ *pRes = 0; rc = sqlite3BtreeNext(pCur, 0); if( rc==SQLITE_OK ){@@ -68104,29 +68777,31 @@ ** bytes of padding is allocated at the end of the buffer in
** case this happens. */ void *pCellKey; u8 * const pCellBody = pCell - pPage->childPtrSize; + const int nOverrun = 18; /* Size of the overrun padding */ pPage->xParseCell(pPage, pCellBody, &pCur->info); nCell = (int)pCur->info.nKey; testcase( nCell<0 ); /* True if key size is 2^32 or more */ testcase( nCell==0 ); /* Invalid key size: 0x80 0x80 0x00 */ testcase( nCell==1 ); /* Invalid key size: 0x80 0x80 0x01 */ testcase( nCell==2 ); /* Minimum legal index key size */ - if( nCell<2 ){ + if( nCell<2 || nCell/pCur->pBt->usableSize>pCur->pBt->nPage ){ rc = SQLITE_CORRUPT_PAGE(pPage); goto moveto_finish; } - pCellKey = sqlite3Malloc( nCell+18 ); + pCellKey = sqlite3Malloc( nCell+nOverrun ); if( pCellKey==0 ){ rc = SQLITE_NOMEM_BKPT; goto moveto_finish; } pCur->ix = (u16)idx; rc = accessPayload(pCur, 0, nCell, (unsigned char*)pCellKey, 0); + memset(((u8*)pCellKey)+nCell,0,nOverrun); /* Fix uninit warnings */ pCur->curFlags &= ~BTCF_ValidOvfl; if( rc ){ sqlite3_free(pCellKey); goto moveto_finish; } - c = xRecordCompare(nCell, pCellKey, pIdxKey); + c = sqlite3VdbeRecordCompare(nCell, pCellKey, pIdxKey); sqlite3_free(pCellKey); } assert(@@ -68242,7 +68917,6 @@ int idx;
MemPage *pPage; assert( cursorOwnsBtShared(pCur) ); - assert( pCur->skipNext==0 || pCur->eState!=CURSOR_VALID ); if( pCur->eState!=CURSOR_VALID ){ assert( (pCur->curFlags & BTCF_ValidOvfl)==0 ); rc = restoreCursorPosition(pCur);@@ -68252,14 +68926,9 @@ }
if( CURSOR_INVALID==pCur->eState ){ return SQLITE_DONE; } - if( pCur->skipNext ){ - assert( pCur->eState==CURSOR_VALID || pCur->eState==CURSOR_SKIPNEXT ); + if( pCur->eState==CURSOR_SKIPNEXT ){ pCur->eState = CURSOR_VALID; - if( pCur->skipNext>0 ){ - pCur->skipNext = 0; - return SQLITE_OK; - } - pCur->skipNext = 0; + if( pCur->skipNext>0 ) return SQLITE_OK; } }@@ -68314,7 +68983,6 @@ MemPage *pPage;
UNUSED_PARAMETER( flags ); /* Used in COMDB2 but not native SQLite */ assert( cursorOwnsBtShared(pCur) ); assert( flags==0 || flags==1 ); - assert( pCur->skipNext==0 || pCur->eState!=CURSOR_VALID ); pCur->info.nSize = 0; pCur->curFlags &= ~(BTCF_ValidNKey|BTCF_ValidOvfl); if( pCur->eState!=CURSOR_VALID ) return btreeNext(pCur);@@ -68355,7 +69023,6 @@ int rc;
MemPage *pPage; assert( cursorOwnsBtShared(pCur) ); - assert( pCur->skipNext==0 || pCur->eState!=CURSOR_VALID ); assert( (pCur->curFlags & (BTCF_AtLast|BTCF_ValidOvfl|BTCF_ValidNKey))==0 ); assert( pCur->info.nSize==0 ); if( pCur->eState!=CURSOR_VALID ){@@ -68366,14 +69033,9 @@ }
if( CURSOR_INVALID==pCur->eState ){ return SQLITE_DONE; } - if( pCur->skipNext ){ - assert( pCur->eState==CURSOR_VALID || pCur->eState==CURSOR_SKIPNEXT ); + if( CURSOR_SKIPNEXT==pCur->eState ){ pCur->eState = CURSOR_VALID; - if( pCur->skipNext<0 ){ - pCur->skipNext = 0; - return SQLITE_OK; - } - pCur->skipNext = 0; + if( pCur->skipNext<0 ) return SQLITE_OK; } }@@ -68408,7 +69070,6 @@ }
SQLITE_PRIVATE int sqlite3BtreePrevious(BtCursor *pCur, int flags){ assert( cursorOwnsBtShared(pCur) ); assert( flags==0 || flags==1 ); - assert( pCur->skipNext==0 || pCur->eState!=CURSOR_VALID ); UNUSED_PARAMETER( flags ); /* Used in COMDB2 but not native SQLite */ pCur->curFlags &= ~(BTCF_AtLast|BTCF_ValidOvfl|BTCF_ValidNKey); pCur->info.nSize = 0;@@ -68744,7 +69405,7 @@ }
TRACE(("ALLOCATE: %d from end of file\n", *pPgno)); } - assert( *pPgno!=PENDING_BYTE_PAGE(pBt) ); + assert( CORRUPT_DB || *pPgno!=PENDING_BYTE_PAGE(pBt) ); end_allocate_page: releasePage(pTrunk);@@ -68772,13 +69433,15 @@ Pgno iTrunk = 0; /* Page number of free-list trunk page */
MemPage *pPage1 = pBt->pPage1; /* Local reference to page 1 */ MemPage *pPage; /* Page being freed. May be NULL. */ int rc; /* Return Code */ - int nFree; /* Initial number of pages on free-list */ + u32 nFree; /* Initial number of pages on free-list */ assert( sqlite3_mutex_held(pBt->mutex) ); assert( CORRUPT_DB || iPage>1 ); assert( !pMemPage || pMemPage->pgno==iPage ); - if( iPage<2 ) return SQLITE_CORRUPT_BKPT; + if( iPage<2 || iPage>pBt->nPage ){ + return SQLITE_CORRUPT_BKPT; + } if( pMemPage ){ pPage = pMemPage; sqlite3PagerRef(pPage->pDbPage);@@ -69189,6 +69852,7 @@ assert( idx>=0 && idx<pPage->nCell );
assert( CORRUPT_DB || sz==cellSize(pPage, idx) ); assert( sqlite3PagerIswriteable(pPage->pDbPage) ); assert( sqlite3_mutex_held(pPage->pBt->mutex) ); + assert( pPage->nFree>=0 ); data = pPage->aData; ptr = &pPage->aCellIdx[2*idx]; pc = get2byte(ptr);@@ -69259,6 +69923,7 @@ ** wanted to be less than 4 but got rounded up to 4 on the leaf, then size
** might be less than 8 (leaf-size + pointer) on the interior node. Hence ** the term after the || in the following assert(). */ assert( sz==pPage->xCellSize(pPage, pCell) || (sz==8 && iChild>0) ); + assert( pPage->nFree>=0 ); if( pPage->nOverflow || sz+2>pPage->nFree ){ if( pTemp ){ memcpy(pTemp, pCell, sz);@@ -69299,9 +69964,16 @@ assert( idx >= 0 );
assert( idx >= pPage->cellOffset+2*pPage->nCell+2 || CORRUPT_DB ); assert( idx+sz <= (int)pPage->pBt->usableSize ); pPage->nFree -= (u16)(2 + sz); - memcpy(&data[idx], pCell, sz); if( iChild ){ + /* In a corrupt database where an entry in the cell index section of + ** a btree page has a value of 3 or less, the pCell value might point + ** as many as 4 bytes in front of the start of the aData buffer for + ** the source page. Make sure this does not cause problems by not + ** reading the first 4 bytes */ + memcpy(&data[idx+4], pCell+4, sz-4); put4byte(&data[idx], iChild); + }else{ + memcpy(&data[idx], pCell, sz); } pIns = pPage->aCellIdx + i*2; memmove(pIns+2, pIns, 2*(pPage->nCell - i));@@ -69309,21 +69981,100 @@ put2byte(pIns, idx);
pPage->nCell++; /* increment the cell count */ if( (++data[pPage->hdrOffset+4])==0 ) data[pPage->hdrOffset+3]++; - assert( get2byte(&data[pPage->hdrOffset+3])==pPage->nCell ); + assert( get2byte(&data[pPage->hdrOffset+3])==pPage->nCell || CORRUPT_DB ); #ifndef SQLITE_OMIT_AUTOVACUUM if( pPage->pBt->autoVacuum ){ /* The cell may contain a pointer to an overflow page. If so, write ** the entry for the overflow page into the pointer map. */ - ptrmapPutOvflPtr(pPage, pCell, pRC); + ptrmapPutOvflPtr(pPage, pPage, pCell, pRC); } #endif } } /* +** The following parameters determine how many adjacent pages get involved +** in a balancing operation. NN is the number of neighbors on either side +** of the page that participate in the balancing operation. NB is the +** total number of pages that participate, including the target page and +** NN neighbors on either side. +** +** The minimum value of NN is 1 (of course). Increasing NN above 1 +** (to 2 or 3) gives a modest improvement in SELECT and DELETE performance +** in exchange for a larger degradation in INSERT and UPDATE performance. +** The value of NN appears to give the best results overall. +** +** (Later:) The description above makes it seem as if these values are +** tunable - as if you could change them and recompile and it would all work. +** But that is unlikely. NB has been 3 since the inception of SQLite and +** we have never tested any other value. +*/ +#define NN 1 /* Number of neighbors on either side of pPage */ +#define NB 3 /* (NN*2+1): Total pages involved in the balance */ + +/* ** A CellArray object contains a cache of pointers and sizes for a ** consecutive sequence of cells that might be held on multiple pages. +** +** The cells in this array are the divider cell or cells from the pParent +** page plus up to three child pages. There are a total of nCell cells. +** +** pRef is a pointer to one of the pages that contributes cells. This is +** used to access information such as MemPage.intKey and MemPage.pBt->pageSize +** which should be common to all pages that contribute cells to this array. +** +** apCell[] and szCell[] hold, respectively, pointers to the start of each +** cell and the size of each cell. Some of the apCell[] pointers might refer +** to overflow cells. In other words, some apCel[] pointers might not point +** to content area of the pages. +** +** A szCell[] of zero means the size of that cell has not yet been computed. +** +** The cells come from as many as four different pages: +** +** ----------- +** | Parent | +** ----------- +** / | \ +** / | \ +** --------- --------- --------- +** |Child-1| |Child-2| |Child-3| +** --------- --------- --------- +** +** The order of cells is in the array is for an index btree is: +** +** 1. All cells from Child-1 in order +** 2. The first divider cell from Parent +** 3. All cells from Child-2 in order +** 4. The second divider cell from Parent +** 5. All cells from Child-3 in order +** +** For a table-btree (with rowids) the items 2 and 4 are empty because +** content exists only in leaves and there are no divider cells. +** +** For an index btree, the apEnd[] array holds pointer to the end of page +** for Child-1, the Parent, Child-2, the Parent (again), and Child-3, +** respectively. The ixNx[] array holds the number of cells contained in +** each of these 5 stages, and all stages to the left. Hence: +** +** ixNx[0] = Number of cells in Child-1. +** ixNx[1] = Number of cells in Child-1 plus 1 for first divider. +** ixNx[2] = Number of cells in Child-1 and Child-2 + 1 for 1st divider. +** ixNx[3] = Number of cells in Child-1 and Child-2 + both divider cells +** ixNx[4] = Total number of cells. +** +** For a table-btree, the concept is similar, except only apEnd[0]..apEnd[2] +** are used and they point to the leaf pages only, and the ixNx value are: +** +** ixNx[0] = Number of cells in Child-1. +** ixNx[1] = Number of cells in Child-1 and Child-2. +** ixNx[2] = Total number of cells. +** +** Sometimes when deleting, a child page can have zero cells. In those +** cases, ixNx[] entries with higher indexes, and the corresponding apEnd[] +** entries, shift down. The end result is that each ixNx[] entry should +** be larger than the previous */ typedef struct CellArray CellArray; struct CellArray {@@ -69331,6 +70082,8 @@ int nCell; /* Number of cells in apCell[] */
MemPage *pRef; /* Reference page */ u8 **apCell; /* All cells begin balanced */ u16 *szCell; /* Local size of all cells in apCell[] */ + u8 *apEnd[NB*2]; /* MemPage.aDataEnd values */ + int ixNx[NB*2]; /* Index of at which we move to the next apEnd[] */ }; /*@@ -69381,36 +70134,59 @@ ** The MemPage.nFree field is invalidated by this function. It is the
** responsibility of the caller to set it correctly. */ static int rebuildPage( - MemPage *pPg, /* Edit this page */ + CellArray *pCArray, /* Content to be added to page pPg */ + int iFirst, /* First cell in pCArray to use */ int nCell, /* Final number of cells on page */ - u8 **apCell, /* Array of cells */ - u16 *szCell /* Array of cell sizes */ + MemPage *pPg /* The page to be reconstructed */ ){ const int hdr = pPg->hdrOffset; /* Offset of header on pPg */ u8 * const aData = pPg->aData; /* Pointer to data for pPg */ const int usableSize = pPg->pBt->usableSize; u8 * const pEnd = &aData[usableSize]; - int i; + int i = iFirst; /* Which cell to copy from pCArray*/ + u32 j; /* Start of cell content area */ + int iEnd = i+nCell; /* Loop terminator */ u8 *pCellptr = pPg->aCellIdx; u8 *pTmp = sqlite3PagerTempSpace(pPg->pBt->pPager); u8 *pData; + int k; /* Current slot in pCArray->apEnd[] */ + u8 *pSrcEnd; /* Current pCArray->apEnd[k] value */ - i = get2byte(&aData[hdr+5]); - memcpy(&pTmp[i], &aData[i], usableSize - i); + assert( i<iEnd ); + j = get2byte(&aData[hdr+5]); + if( NEVER(j>(u32)usableSize) ){ j = 0; } + memcpy(&pTmp[j], &aData[j], usableSize - j); + + for(k=0; pCArray->ixNx[k]<=i && ALWAYS(k<NB*2); k++){} + pSrcEnd = pCArray->apEnd[k]; pData = pEnd; - for(i=0; i<nCell; i++){ - u8 *pCell = apCell[i]; + while( 1/*exit by break*/ ){ + u8 *pCell = pCArray->apCell[i]; + u16 sz = pCArray->szCell[i]; + assert( sz>0 ); if( SQLITE_WITHIN(pCell,aData,pEnd) ){ + if( ((uptr)(pCell+sz))>(uptr)pEnd ) return SQLITE_CORRUPT_BKPT; pCell = &pTmp[pCell - aData]; + }else if( (uptr)(pCell+sz)>(uptr)pSrcEnd + && (uptr)(pCell)<(uptr)pSrcEnd + ){ + return SQLITE_CORRUPT_BKPT; } - pData -= szCell[i]; + + pData -= sz; put2byte(pCellptr, (pData - aData)); pCellptr += 2; if( pData < pCellptr ) return SQLITE_CORRUPT_BKPT; - memcpy(pData, pCell, szCell[i]); - assert( szCell[i]==pPg->xCellSize(pPg, pCell) || CORRUPT_DB ); - testcase( szCell[i]!=pPg->xCellSize(pPg,pCell) ); + memcpy(pData, pCell, sz); + assert( sz==pPg->xCellSize(pPg, pCell) || CORRUPT_DB ); + testcase( sz!=pPg->xCellSize(pPg,pCell) ); + i++; + if( i>=iEnd ) break; + if( pCArray->ixNx[k]<=i ){ + k++; + pSrcEnd = pCArray->apEnd[k]; + } } /* The pPg->nFree field is now set incorrectly. The caller will fix it. */@@ -69425,12 +70201,11 @@ return SQLITE_OK;
} /* -** Array apCell[] contains nCell pointers to b-tree cells. Array szCell -** contains the size in bytes of each such cell. This function attempts to -** add the cells stored in the array to page pPg. If it cannot (because -** the page needs to be defragmented before the cells will fit), non-zero -** is returned. Otherwise, if the cells are added successfully, zero is -** returned. +** The pCArray objects contains pointers to b-tree cells and the cell sizes. +** This function attempts to add the cells stored in the array to page pPg. +** If it cannot (because the page needs to be defragmented before the cells +** will fit), non-zero is returned. Otherwise, if the cells are added +** successfully, zero is returned. ** ** Argument pCellptr points to the first entry in the cell-pointer array ** (part of page pPg) to populate. After cell apCell[0] is written to the@@ -69452,18 +70227,23 @@ */
static int pageInsertArray( MemPage *pPg, /* Page to add cells to */ u8 *pBegin, /* End of cell-pointer array */ - u8 **ppData, /* IN/OUT: Page content -area pointer */ + u8 **ppData, /* IN/OUT: Page content-area pointer */ u8 *pCellptr, /* Pointer to cell-pointer area */ int iFirst, /* Index of first cell to add */ int nCell, /* Number of cells to add to pPg */ CellArray *pCArray /* Array of cells */ ){ - int i; - u8 *aData = pPg->aData; - u8 *pData = *ppData; - int iEnd = iFirst + nCell; + int i = iFirst; /* Loop counter - cell index to insert */ + u8 *aData = pPg->aData; /* Complete page */ + u8 *pData = *ppData; /* Content area. A subset of aData[] */ + int iEnd = iFirst + nCell; /* End of loop. One past last cell to ins */ + int k; /* Current slot in pCArray->apEnd[] */ + u8 *pEnd; /* Maximum extent of cell data */ assert( CORRUPT_DB || pPg->hdrOffset==0 ); /* Never called on page 1 */ - for(i=iFirst; i<iEnd; i++){ + if( iEnd<=iFirst ) return 0; + for(k=0; pCArray->ixNx[k]<=i && ALWAYS(k<NB*2); k++){} + pEnd = pCArray->apEnd[k]; + while( 1 /*Exit by break*/ ){ int sz, rc; u8 *pSlot; sz = cachedCellSize(pCArray, i);@@ -69478,20 +70258,33 @@ ** since memcpy() sends SIGABORT with overlapping buffers on OpenBSD */
assert( (pSlot+sz)<=pCArray->apCell[i] || pSlot>=(pCArray->apCell[i]+sz) || CORRUPT_DB ); + if( (uptr)(pCArray->apCell[i]+sz)>(uptr)pEnd + && (uptr)(pCArray->apCell[i])<(uptr)pEnd + ){ + assert( CORRUPT_DB ); + (void)SQLITE_CORRUPT_BKPT; + return 1; + } memmove(pSlot, pCArray->apCell[i], sz); put2byte(pCellptr, (pSlot - aData)); pCellptr += 2; + i++; + if( i>=iEnd ) break; + if( pCArray->ixNx[k]<=i ){ + k++; + pEnd = pCArray->apEnd[k]; + } } *ppData = pData; return 0; } /* -** Array apCell[] contains nCell pointers to b-tree cells. Array szCell -** contains the size in bytes of each such cell. This function adds the -** space associated with each cell in the array that is currently stored -** within the body of pPg to the pPg free-list. The cell-pointers and other -** fields of the page are not updated. +** The pCArray object contains pointers to b-tree cells and their sizes. +** +** This function adds the space associated with each cell in the array +** that is currently stored within the body of pPg to the pPg free-list. +** The cell-pointers and other fields of the page are not updated. ** ** This function returns the total number of cells added to the free-list. */@@ -69541,9 +70334,9 @@ return nRet;
} /* -** apCell[] and szCell[] contains pointers to and sizes of all cells in the -** pages being balanced. The current page, pPg, has pPg->nCell cells starting -** with apCell[iOld]. After balancing, this page should hold nNew cells +** pCArray contains pointers to and sizes of all cells in the page being +** balanced. The current page, pPg, has pPg->nCell cells starting with +** pCArray->apCell[iOld]. After balancing, this page should hold nNew cells ** starting at apCell[iNew]. ** ** This routine makes the necessary adjustments to pPg so that it contains@@ -69575,13 +70368,17 @@ memcpy(pTmp, aData, pPg->pBt->usableSize);
#endif /* Remove cells from the start and end of the page */ + assert( nCell>=0 ); if( iOld<iNew ){ int nShift = pageFreeArray(pPg, iOld, iNew-iOld, pCArray); + if( nShift>nCell ) return SQLITE_CORRUPT_BKPT; memmove(pPg->aCellIdx, &pPg->aCellIdx[nShift*2], nCell*2); nCell -= nShift; } if( iNewEnd < iOldEnd ){ - nCell -= pageFreeArray(pPg, iNewEnd, iOldEnd - iNewEnd, pCArray); + int nTail = pageFreeArray(pPg, iNewEnd, iOldEnd - iNewEnd, pCArray); + assert( nCell>=nTail ); + nCell -= nTail; } pData = &aData[get2byteNotZero(&aData[hdr+5])];@@ -69591,6 +70388,7 @@ /* Add cells to the start of the page */
if( iNew<iOld ){ int nAdd = MIN(nNew,iOld-iNew); assert( (iOld-iNew)<nNew || nCell==0 || CORRUPT_DB ); + assert( nAdd>=0 ); pCellptr = pPg->aCellIdx; memmove(&pCellptr[nAdd*2], pCellptr, nCell*2); if( pageInsertArray(@@ -69605,7 +70403,9 @@ for(i=0; i<pPg->nOverflow; i++){
int iCell = (iOld + pPg->aiOvfl[i]) - iNew; if( iCell>=0 && iCell<nNew ){ pCellptr = &pPg->aCellIdx[iCell * 2]; - memmove(&pCellptr[2], pCellptr, (nCell - iCell) * 2); + if( nCell>iCell ){ + memmove(&pCellptr[2], pCellptr, (nCell - iCell) * 2); + } nCell++; if( pageInsertArray( pPg, pBegin, &pData, pCellptr,@@ -69615,6 +70415,7 @@ }
} /* Append cells to the end of the page */ + assert( nCell>=0 ); pCellptr = &pPg->aCellIdx[nCell*2]; if( pageInsertArray( pPg, pBegin, &pData, pCellptr,@@ -69643,24 +70444,9 @@ return SQLITE_OK;
editpage_fail: /* Unable to edit this page. Rebuild it from scratch instead. */ populateCellCache(pCArray, iNew, nNew); - return rebuildPage(pPg, nNew, &pCArray->apCell[iNew], &pCArray->szCell[iNew]); + return rebuildPage(pCArray, iNew, nNew, pPg); } -/* -** The following parameters determine how many adjacent pages get involved -** in a balancing operation. NN is the number of neighbors on either side -** of the page that participate in the balancing operation. NB is the -** total number of pages that participate, including the target page and -** NN neighbors on either side. -** -** The minimum value of NN is 1 (of course). Increasing NN above 1 -** (to 2 or 3) gives a modest improvement in SELECT and DELETE performance -** in exchange for a larger degradation in INSERT and UPDATE performance. -** The value of NN appears to give the best results overall. -*/ -#define NN 1 /* Number of neighbors on either side of pPage */ -#define NB (NN*2+1) /* Total pages involved in the balance */ - #ifndef SQLITE_OMIT_QUICKBALANCE /*@@ -69695,9 +70481,10 @@
assert( sqlite3_mutex_held(pPage->pBt->mutex) ); assert( sqlite3PagerIswriteable(pParent->pDbPage) ); assert( pPage->nOverflow==1 ); - - /* This error condition is now caught prior to reaching this function */ - if( NEVER(pPage->nCell==0) ) return SQLITE_CORRUPT_BKPT; + + if( pPage->nCell==0 ) return SQLITE_CORRUPT_BKPT; /* dbfuzz001.test */ + assert( pPage->nFree>=0 ); + assert( pParent->nFree>=0 ); /* Allocate a new page. This page will become the right-sibling of ** pPage. Make the parent page writable, so that the new divider cell@@ -69711,12 +70498,22 @@ u8 *pOut = &pSpace[4];
u8 *pCell = pPage->apOvfl[0]; u16 szCell = pPage->xCellSize(pPage, pCell); u8 *pStop; + CellArray b; assert( sqlite3PagerIswriteable(pNew->pDbPage) ); - assert( pPage->aData[0]==(PTF_INTKEY|PTF_LEAFDATA|PTF_LEAF) ); + assert( CORRUPT_DB || pPage->aData[0]==(PTF_INTKEY|PTF_LEAFDATA|PTF_LEAF) ); zeroPage(pNew, PTF_INTKEY|PTF_LEAFDATA|PTF_LEAF); - rc = rebuildPage(pNew, 1, &pCell, &szCell); - if( NEVER(rc) ) return rc; + b.nCell = 1; + b.pRef = pPage; + b.apCell = &pCell; + b.szCell = &szCell; + b.apEnd[0] = pPage->aDataEnd; + b.ixNx[0] = 2; + rc = rebuildPage(&b, 0, 1, pNew); + if( NEVER(rc) ){ + releasePage(pNew); + return rc; + } pNew->nFree = pBt->usableSize - pNew->cellOffset - 2 - szCell; /* If this is an auto-vacuum database, update the pointer map@@ -69731,7 +70528,7 @@ */
if( ISAUTOVACUUM ){ ptrmapPut(pBt, pgnoNew, PTRMAP_BTREE, pParent->pgno, &rc); if( szCell>pNew->minLocal ){ - ptrmapPutOvflPtr(pNew, pCell, &rc); + ptrmapPutOvflPtr(pNew, pNew, pCell, &rc); } }@@ -69857,6 +70654,7 @@ ** page pFrom.
*/ pTo->isInit = 0; rc = btreeInitPage(pTo); + if( rc==SQLITE_OK ) rc = btreeComputeFreeSpace(pTo); if( rc!=SQLITE_OK ){ *pRC = rc; return;@@ -69954,10 +70752,6 @@ pBt = pParent->pBt;
assert( sqlite3_mutex_held(pBt->mutex) ); assert( sqlite3PagerIswriteable(pParent->pDbPage) ); -#if 0 - TRACE(("BALANCE: begin page %d child of %d\n", pPage->pgno, pParent->pgno)); -#endif - /* At this point pParent may have at most one overflow cell. And if ** this overflow cell is present, it must be the cell with ** index iParentIdx. This scenario comes about when this function@@ -69969,6 +70763,7 @@
if( !aOvflSpace ){ return SQLITE_NOMEM_BKPT; } + assert( pParent->nFree>=0 ); /* Find the sibling pages to balance. Also locate the cells in pParent ** that divide the siblings. An attempt is made to find NN siblings on@@ -70008,7 +70803,13 @@ if( rc ){
memset(apOld, 0, (i+1)*sizeof(MemPage*)); goto balance_cleanup; } - nMaxCells += 1+apOld[i]->nCell+apOld[i]->nOverflow; + if( apOld[i]->nFree<0 ){ + rc = btreeComputeFreeSpace(apOld[i]); + if( rc ){ + memset(apOld, 0, (i)*sizeof(MemPage*)); + goto balance_cleanup; + } + } if( (i--)==0 ) break; if( pParent->nOverflow && i+nxDiv==pParent->aiOvfl[0] ){@@ -70052,6 +70853,7 @@ }
/* Make nMaxCells a multiple of 4 in order to preserve 8-byte ** alignment */ + nMaxCells = nOld*(MX_CELL(pBt) + ArraySize(pParent->apOvfl)); nMaxCells = (nMaxCells + 3)&~3; /*@@ -70062,7 +70864,7 @@ nMaxCells*sizeof(u8*) /* b.apCell */
+ nMaxCells*sizeof(u16) /* b.szCell */ + pBt->pageSize; /* aSpace1 */ - assert( szScratch<=6*(int)pBt->pageSize ); + assert( szScratch<=7*(int)pBt->pageSize ); b.apCell = sqlite3StackAllocRaw(0, szScratch ); if( b.apCell==0 ){ rc = SQLITE_NOMEM_BKPT;@@ -70098,6 +70900,7 @@ u8 *aData = pOld->aData;
u16 maskPage = pOld->maskPage; u8 *piCell = aData + pOld->cellOffset; u8 *piEnd; + VVA_ONLY( int nCellAtStart = b.nCell; ) /* Verify that all sibling pages are of the same "type" (table-leaf, ** table-interior, index-leaf, or index-interior).@@ -70126,6 +70929,10 @@ ** first.
*/ memset(&b.szCell[b.nCell], 0, sizeof(b.szCell[0])*(limit+pOld->nOverflow)); if( pOld->nOverflow>0 ){ + if( limit<pOld->aiOvfl[0] ){ + rc = SQLITE_CORRUPT_BKPT; + goto balance_cleanup; + } limit = pOld->aiOvfl[0]; for(j=0; j<limit; j++){ b.apCell[b.nCell] = aData + (maskPage & get2byteAligned(piCell));@@ -70145,6 +70952,7 @@ b.apCell[b.nCell] = aData + (maskPage & get2byteAligned(piCell));
piCell += 2; b.nCell++; } + assert( (b.nCell-nCellAtStart)==(pOld->nCell+pOld->nOverflow) ); cntOld[i] = b.nCell; if( i<nOld-1 && !leafData){@@ -70198,8 +71006,19 @@ ** usableSpace: Number of bytes of space available on each sibling.
** */ usableSpace = pBt->usableSize - 12 + leafCorrection; - for(i=0; i<nOld; i++){ + for(i=k=0; i<nOld; i++, k++){ MemPage *p = apOld[i]; + b.apEnd[k] = p->aDataEnd; + b.ixNx[k] = cntOld[i]; + if( k && b.ixNx[k]==b.ixNx[k-1] ){ + k--; /* Omit b.ixNx[] entry for child pages with no cells */ + } + if( !leafData ){ + k++; + b.apEnd[k] = pParent->aDataEnd; + b.ixNx[k] = cntOld[i]+1; + } + assert( p->nFree>=0 ); szNew[i] = usableSpace - p->nFree; for(j=0; j<p->nOverflow; j++){ szNew[i] += 2 + p->xCellSize(p, p->apOvfl[j]);@@ -70423,19 +71242,20 @@ ** updated. This happens below, after the sibling pages have been
** populated, not here. */ if( ISAUTOVACUUM ){ - MemPage *pNew = apNew[0]; - u8 *aOld = pNew->aData; + MemPage *pOld; + MemPage *pNew = pOld = apNew[0]; int cntOldNext = pNew->nCell + pNew->nOverflow; - int usableSize = pBt->usableSize; int iNew = 0; int iOld = 0; for(i=0; i<b.nCell; i++){ u8 *pCell = b.apCell[i]; - if( i==cntOldNext ){ - MemPage *pOld = (++iOld)<nNew ? apNew[iOld] : apOld[iOld]; + while( i==cntOldNext ){ + iOld++; + assert( iOld<nNew || iOld<nOld ); + assert( iOld>=0 && iOld<NB ); + pOld = iOld<nNew ? apNew[iOld] : apOld[iOld]; cntOldNext += pOld->nCell + pOld->nOverflow + !leafData; - aOld = pOld->aData; } if( i==cntNew[iNew] ){ pNew = apNew[++iNew];@@ -70450,13 +71270,13 @@ ** pCell really was a part of sibling page iOld (not a divider or
** overflow cell), we can skip updating the pointer map entries. */ if( iOld>=nNew || pNew->pgno!=aPgno[iOld] - || !SQLITE_WITHIN(pCell,aOld,&aOld[usableSize]) + || !SQLITE_WITHIN(pCell,pOld->aData,pOld->aDataEnd) ){ if( !leafCorrection ){ ptrmapPut(pBt, get4byte(pCell), PTRMAP_BTREE, pNew->pgno, &rc); } if( cachedCellSize(&b,i)>pNew->minLocal ){ - ptrmapPutOvflPtr(pNew, pCell, &rc); + ptrmapPutOvflPtr(pNew, pOld, pCell, &rc); } if( rc ) goto balance_cleanup; }@@ -70601,7 +71421,8 @@ assert( nNew==1 || CORRUPT_DB );
rc = defragmentPage(apNew[0], -1); testcase( rc!=SQLITE_OK ); assert( apNew[0]->nFree == - (get2byte(&apNew[0]->aData[5])-apNew[0]->cellOffset-apNew[0]->nCell*2) + (get2byteNotZero(&apNew[0]->aData[5]) - apNew[0]->cellOffset + - apNew[0]->nCell*2) || rc!=SQLITE_OK ); copyNodeContent(apNew[0], pParent, &rc);@@ -70700,7 +71521,7 @@ return rc;
} assert( sqlite3PagerIswriteable(pChild->pDbPage) ); assert( sqlite3PagerIswriteable(pRoot->pDbPage) ); - assert( pChild->nCell==pRoot->nCell ); + assert( pChild->nCell==pRoot->nCell || CORRUPT_DB ); TRACE(("BALANCE: copy root %d into %d\n", pRoot->pgno, pChild->pgno));@@ -70742,6 +71563,7 @@ do {
int iPage = pCur->iPage; MemPage *pPage = pCur->pPage; + if( NEVER(pPage->nFree<0) && btreeComputeFreeSpace(pPage) ) break; if( iPage==0 ){ if( pPage->nOverflow ){ /* The root page of the b-tree is overfull. In this case call the@@ -70770,6 +71592,9 @@ MemPage * const pParent = pCur->apPage[iPage-1];
int const iIdx = pCur->aiIdx[iPage-1]; rc = sqlite3PagerWrite(pParent->pDbPage); + if( rc==SQLITE_OK && pParent->nFree<0 ){ + rc = btreeComputeFreeSpace(pParent); + } if( rc==SQLITE_OK ){ #ifndef SQLITE_OMIT_QUICKBALANCE if( pPage->intKeyLeaf@@ -70880,7 +71705,11 @@ }
if( memcmp(pDest, ((u8*)pX->pData) + iOffset, iAmt)!=0 ){ int rc = sqlite3PagerWrite(pPage->pDbPage); if( rc ) return rc; - memcpy(pDest, ((u8*)pX->pData) + iOffset, iAmt); + /* In a corrupt database, it is possible for the source and destination + ** buffers to overlap. This is harmless since the database is already + ** corrupt but it does cause valgrind and ASAN warnings. So use + ** memmove(). */ + memmove(pDest, ((u8*)pX->pData) + iOffset, iAmt); } } return SQLITE_OK;@@ -71112,6 +71941,10 @@
pPage = pCur->pPage; assert( pPage->intKey || pX->nKey>=0 ); assert( pPage->leaf || !pPage->intKey ); + if( pPage->nFree<0 ){ + rc = btreeComputeFreeSpace(pPage); + if( rc ) return rc; + } TRACE(("INSERT: table=%d nkey=%lld ndata=%d page=%d %s\n", pCur->pgnoRoot, pX->nKey, pX->nData, pPage->pgno,@@ -71254,14 +72087,18 @@ assert( (pBt->btsFlags & BTS_READ_ONLY)==0 );
assert( pCur->curFlags & BTCF_WriteFlag ); assert( hasSharedCacheTableLock(p, pCur->pgnoRoot, pCur->pKeyInfo!=0, 2) ); assert( !hasReadConflicts(p, pCur->pgnoRoot) ); - assert( pCur->ix<pCur->pPage->nCell ); + assert( (flags & ~(BTREE_SAVEPOSITION | BTREE_AUXDELETE))==0 ); + if( pCur->eState==CURSOR_REQUIRESEEK ){ + rc = btreeRestoreCursorPosition(pCur); + if( rc ) return rc; + } assert( pCur->eState==CURSOR_VALID ); - assert( (flags & ~(BTREE_SAVEPOSITION | BTREE_AUXDELETE))==0 ); iCellDepth = pCur->iPage; iCellIdx = pCur->ix; pPage = pCur->pPage; pCell = findCell(pPage, iCellIdx); + if( pPage->nFree<0 && btreeComputeFreeSpace(pPage) ) return SQLITE_CORRUPT; /* If the bPreserve flag is set to true, then the cursor position must ** be preserved following this delete operation. If the current delete@@ -71275,6 +72112,7 @@ ** before or after the deleted entry. In this case set bSkipnext to true. */
if( bPreserve ){ if( !pPage->leaf || (pPage->nFree+cellSizePtr(pPage,pCell)+2)>(int)(pBt->usableSize*2/3) + || pPage->nCell==1 /* See dbfuzz001.test for a test case */ ){ /* A b-tree rebalance will be required after deleting this entry. ** Save the cursor key. */@@ -71331,6 +72169,10 @@ int nCell;
Pgno n; unsigned char *pTmp; + if( pLeaf->nFree<0 ){ + rc = btreeComputeFreeSpace(pLeaf); + if( rc ) return rc; + } if( iCellDepth<pCur->iPage-1 ){ n = pCur->apPage[iCellDepth+1]->pgno; }else{@@ -71689,6 +72531,9 @@
assert( sqlite3BtreeHoldsMutex(p) ); assert( p->inTrans==TRANS_WRITE ); assert( iTable>=2 ); + if( iTable>btreePagecount(pBt) ){ + return SQLITE_CORRUPT_BKPT; + } rc = btreeGetPage(pBt, (Pgno)iTable, &pPage, 0); if( rc ) return rc;@@ -72037,10 +72882,10 @@ static void checkList(
IntegrityCk *pCheck, /* Integrity checking context */ int isFreeList, /* True for a freelist. False for overflow page list */ int iPage, /* Page number for first page in the list */ - int N /* Expected number of pages in the list */ + u32 N /* Expected number of pages in the list */ ){ int i; - int expected = N; + u32 expected = N; int nErrAtStart = pCheck->nErr; while( iPage!=0 && pCheck->mxErr ){ DbPage *pOvflPage;@@ -72053,18 +72898,18 @@ break;
} pOvflData = (unsigned char *)sqlite3PagerGetData(pOvflPage); if( isFreeList ){ - int n = get4byte(&pOvflData[4]); + u32 n = (u32)get4byte(&pOvflData[4]); #ifndef SQLITE_OMIT_AUTOVACUUM if( pCheck->pBt->autoVacuum ){ checkPtrmap(pCheck, iPage, PTRMAP_FREEPAGE, 0); } #endif - if( n>(int)pCheck->pBt->usableSize/4-2 ){ + if( n>pCheck->pBt->usableSize/4-2 ){ checkAppendMsg(pCheck, "freelist leaf count too big on page %d", iPage); N--; }else{ - for(i=0; i<n; i++){ + for(i=0; i<(int)n; i++){ Pgno iFreePage = get4byte(&pOvflData[8+i*4]); #ifndef SQLITE_OMIT_AUTOVACUUM if( pCheck->pBt->autoVacuum ){@@ -72222,6 +73067,11 @@ checkAppendMsg(pCheck,
"btreeInitPage() returns error code %d", rc); goto end_of_check; } + if( (rc = btreeComputeFreeSpace(pPage))!=0 ){ + assert( rc==SQLITE_CORRUPT ); + checkAppendMsg(pCheck, "free space corruption", rc); + goto end_of_check; + } data = pPage->aData; hdr = pPage->hdrOffset;@@ -72294,7 +73144,7 @@ }
/* Check the content overflow list */ if( info.nPayload>info.nLocal ){ - int nPage; /* Number of pages on the overflow chain */ + u32 nPage; /* Number of pages on the overflow chain */ Pgno pgnoOvfl; /* First page of the overflow chain */ assert( pc + info.nSize - 4 <= usableSize ); nPage = (info.nPayload - info.nLocal + usableSize - 5)/(usableSize - 4);@@ -72354,9 +73204,9 @@ */
i = get2byte(&data[hdr+1]); while( i>0 ){ int size, j; - assert( (u32)i<=usableSize-4 ); /* Enforced by btreeInitPage() */ + assert( (u32)i<=usableSize-4 ); /* Enforced by btreeComputeFreeSpace() */ size = get2byte(&data[i+2]); - assert( (u32)(i+size)<=usableSize ); /* Enforced by btreeInitPage() */ + assert( (u32)(i+size)<=usableSize ); /* due to btreeComputeFreeSpace() */ btreeHeapInsert(heap, (((u32)i)<<16)|(i+size-1)); /* EVIDENCE-OF: R-58208-19414 The first 2 bytes of a freeblock are a ** big-endian integer which is the offset in the b-tree page of the next@@ -72365,8 +73215,8 @@ ** chain. */
j = get2byte(&data[i]); /* EVIDENCE-OF: R-06866-39125 Freeblocks are always connected in order of ** increasing offset. */ - assert( j==0 || j>i+size ); /* Enforced by btreeInitPage() */ - assert( (u32)j<=usableSize-4 ); /* Enforced by btreeInitPage() */ + assert( j==0 || j>i+size ); /* Enforced by btreeComputeFreeSpace() */ + assert( (u32)j<=usableSize-4 ); /* Enforced by btreeComputeFreeSpace() */ i = j; } /* Analyze the min-heap looking for overlap between cells and/or@@ -72441,7 +73291,7 @@ ){
Pgno i; IntegrityCk sCheck; BtShared *pBt = p->pBt; - int savedDbFlags = pBt->db->flags; + u64 savedDbFlags = pBt->db->flags; char zErr[100]; VVA_ONLY( int nRef );@@ -72508,7 +73358,7 @@ );
} #endif testcase( pBt->db->flags & SQLITE_CellSizeCk ); - pBt->db->flags &= ~SQLITE_CellSizeCk; + pBt->db->flags &= ~(u64)SQLITE_CellSizeCk; for(i=0; (int)i<nRoot && sCheck.mxErr; i++){ i64 notUsed; if( aRoot[i]==0 ) continue;@@ -73124,7 +73974,7 @@ */
if( nSrcReserve!=nDestReserve ){ u32 newPgsz = nSrcPgsz; rc = sqlite3PagerSetPagesize(pDestPager, &newPgsz, nSrcReserve); - if( rc==SQLITE_OK && newPgsz!=nSrcPgsz ) rc = SQLITE_READONLY; + if( rc==SQLITE_OK && newPgsz!=(u32)nSrcPgsz ) rc = SQLITE_READONLY; } #endif@@ -73671,6 +74521,11 @@ */
/* #include "sqliteInt.h" */ /* #include "vdbeInt.h" */ +/* True if X is a power of two. 0 is considered a power of two here. +** In other words, return true if X has at most one bit set. +*/ +#define ISPOWEROF2(X) (((X)&((X)-1))==0) + #ifdef SQLITE_DEBUG /* ** Check invariants on a Mem object.@@ -73690,8 +74545,8 @@ ** Mem.z = Mem.zMalloc without having to check Mem.flags&MEM_Dyn.
** That saves a few cycles in inner loops. */ assert( (p->flags & MEM_Dyn)==0 || p->szMalloc==0 ); - /* Cannot be both MEM_Int and MEM_Real at the same time */ - assert( (p->flags & (MEM_Int|MEM_Real))!=(MEM_Int|MEM_Real) ); + /* Cannot have more than one of MEM_Int, MEM_Real, or MEM_IntReal */ + assert( ISPOWEROF2(p->flags & (MEM_Int|MEM_Real|MEM_IntReal)) ); if( p->flags & MEM_Null ){ /* Cannot be both MEM_Null and some other type */@@ -73710,7 +74565,7 @@ ((p->flags&MEM_Ephem)!=0 ? 1 : 0) +
((p->flags&MEM_Static)!=0 ? 1 : 0) <= 1 ); /* No other bits set */ - assert( (p->flags & ~(MEM_Null|MEM_Term|MEM_Subtype + assert( (p->flags & ~(MEM_Null|MEM_Term|MEM_Subtype|MEM_FromBind |MEM_Dyn|MEM_Ephem|MEM_Static))==0 ); }else{ /* A pure NULL might have other flags, such as MEM_Static, MEM_Dyn,@@ -73745,9 +74600,31 @@ return 1;
} #endif +/* +** Render a Mem object which is one of MEM_Int, MEM_Real, or MEM_IntReal +** into a buffer. +*/ +static void vdbeMemRenderNum(int sz, char *zBuf, Mem *p){ + StrAccum acc; + assert( p->flags & (MEM_Int|MEM_Real|MEM_IntReal) ); + sqlite3StrAccumInit(&acc, 0, zBuf, sz, 0); + if( p->flags & MEM_Int ){ + sqlite3_str_appendf(&acc, "%lld", p->u.i); + }else if( p->flags & MEM_IntReal ){ + sqlite3_str_appendf(&acc, "%!.15g", (double)p->u.i); + }else{ + sqlite3_str_appendf(&acc, "%!.15g", p->u.r); + } + assert( acc.zText==zBuf && acc.mxAlloc<=0 ); + zBuf[acc.nChar] = 0; /* Fast version of sqlite3StrAccumFinish(&acc) */ +} + #ifdef SQLITE_DEBUG /* -** Check that string value of pMem agrees with its integer or real value. +** Validity checks on pMem. pMem holds a string. +** +** (1) Check that string value of pMem agrees with its integer or real value. +** (2) Check that the string is correctly zero terminated ** ** A single int or real value always converts to the same strings. But ** many different strings can be converted into the same int or real.@@ -73765,17 +74642,24 @@ ** true if everything is ok and false if there is a problem.
** ** This routine is for use inside of assert() statements only. */ -SQLITE_PRIVATE int sqlite3VdbeMemConsistentDualRep(Mem *p){ +SQLITE_PRIVATE int sqlite3VdbeMemValidStrRep(Mem *p){ char zBuf[100]; char *z; int i, j, incr; if( (p->flags & MEM_Str)==0 ) return 1; - if( (p->flags & (MEM_Int|MEM_Real))==0 ) return 1; - if( p->flags & MEM_Int ){ - sqlite3_snprintf(sizeof(zBuf),zBuf,"%lld",p->u.i); - }else{ - sqlite3_snprintf(sizeof(zBuf),zBuf,"%!.15g",p->u.r); + if( p->flags & MEM_Term ){ + /* Insure that the string is properly zero-terminated. Pay particular + ** attention to the case where p->n is odd */ + if( p->szMalloc>0 && p->z==p->zMalloc ){ + assert( p->enc==SQLITE_UTF8 || p->szMalloc >= ((p->n+1)&~1)+2 ); + assert( p->enc!=SQLITE_UTF8 || p->szMalloc >= p->n+1 ); + } + assert( p->z[p->n]==0 ); + assert( p->enc==SQLITE_UTF8 || p->z[(p->n+1)&~1]==0 ); + assert( p->enc==SQLITE_UTF8 || p->z[((p->n+1)&~1)+1]==0 ); } + if( (p->flags & (MEM_Int|MEM_Real|MEM_IntReal))==0 ) return 1; + vdbeMemRenderNum(sizeof(zBuf), zBuf, p); z = p->z; i = j = 0; incr = 1;@@ -73831,8 +74715,7 @@ #endif
} /* -** Make sure pMem->z points to a writable allocation of at least -** min(n,32) bytes. +** Make sure pMem->z points to a writable allocation of at least n bytes. ** ** If the bPreserve argument is true, then copy of the content of ** pMem->z into the new allocation. pMem must be either a string or@@ -73851,7 +74734,6 @@ testcase( bPreserve && pMem->z==0 );
assert( pMem->szMalloc==0 || pMem->szMalloc==sqlite3DbMallocSize(pMem->db, pMem->zMalloc) ); - if( n<32 ) n = 32; if( pMem->szMalloc>0 && bPreserve && pMem->z==pMem->zMalloc ){ pMem->z = pMem->zMalloc = sqlite3DbReallocOrFree(pMem->db, pMem->z, n); bPreserve = 0;@@ -73889,34 +74771,40 @@ ** routine is a no-op.
** ** Any prior string or blob content in the pMem object may be discarded. ** The pMem->xDel destructor is called, if it exists. Though MEM_Str -** and MEM_Blob values may be discarded, MEM_Int, MEM_Real, and MEM_Null -** values are preserved. +** and MEM_Blob values may be discarded, MEM_Int, MEM_Real, MEM_IntReal, +** and MEM_Null values are preserved. ** ** Return SQLITE_OK on success or an error code (probably SQLITE_NOMEM) ** if unable to complete the resizing. */ SQLITE_PRIVATE int sqlite3VdbeMemClearAndResize(Mem *pMem, int szNew){ - assert( szNew>0 ); + assert( CORRUPT_DB || szNew>0 ); assert( (pMem->flags & MEM_Dyn)==0 || pMem->szMalloc==0 ); if( pMem->szMalloc<szNew ){ return sqlite3VdbeMemGrow(pMem, szNew, 0); } assert( (pMem->flags & MEM_Dyn)==0 ); pMem->z = pMem->zMalloc; - pMem->flags &= (MEM_Null|MEM_Int|MEM_Real); + pMem->flags &= (MEM_Null|MEM_Int|MEM_Real|MEM_IntReal); return SQLITE_OK; } /* ** It is already known that pMem contains an unterminated string. ** Add the zero terminator. +** +** Three bytes of zero are added. In this way, there is guaranteed +** to be a double-zero byte at an even byte boundary in order to +** terminate a UTF16 string, even if the initial size of the buffer +** is an odd number of bytes. */ static SQLITE_NOINLINE int vdbeMemAddTerminator(Mem *pMem){ - if( sqlite3VdbeMemGrow(pMem, pMem->n+2, 1) ){ + if( sqlite3VdbeMemGrow(pMem, pMem->n+3, 1) ){ return SQLITE_NOMEM_BKPT; } pMem->z[pMem->n] = 0; pMem->z[pMem->n+1] = 0; + pMem->z[pMem->n+2] = 0; pMem->flags |= MEM_Term; return SQLITE_OK; }@@ -73953,13 +74841,15 @@ #ifndef SQLITE_OMIT_INCRBLOB
SQLITE_PRIVATE int sqlite3VdbeMemExpandBlob(Mem *pMem){ int nByte; assert( pMem->flags & MEM_Zero ); - assert( pMem->flags&MEM_Blob ); + assert( (pMem->flags&MEM_Blob)!=0 || MemNullNochng(pMem) ); + testcase( sqlite3_value_nochange(pMem) ); assert( !sqlite3VdbeMemIsRowSet(pMem) ); assert( pMem->db==0 || sqlite3_mutex_held(pMem->db->mutex) ); /* Set nByte to the number of bytes required to store the expanded blob. */ nByte = pMem->n + pMem->u.nZero; if( nByte<=0 ){ + if( (pMem->flags & MEM_Blob)==0 ) return SQLITE_OK; nByte = 1; } if( sqlite3VdbeMemGrow(pMem, nByte, 1) ){@@ -73988,12 +74878,12 @@ }
} /* -** Add MEM_Str to the set of representations for the given Mem. Numbers -** are converted using sqlite3_snprintf(). Converting a BLOB to a string -** is a no-op. +** Add MEM_Str to the set of representations for the given Mem. This +** routine is only called if pMem is a number of some kind, not a NULL +** or a BLOB. ** -** Existing representations MEM_Int and MEM_Real are invalidated if -** bForce is true but are retained if bForce is false. +** Existing representations MEM_Int, MEM_Real, or MEM_IntReal are invalidated +** if bForce is true but are retained if bForce is false. ** ** A MEM_Null value will never be passed to this function. This function is ** used for converting values to text for returning to the user (i.e. via@@ -74002,13 +74892,12 @@ ** keys are strings. In the former case a NULL pointer is returned the
** user and the latter is an internal programming error. */ SQLITE_PRIVATE int sqlite3VdbeMemStringify(Mem *pMem, u8 enc, u8 bForce){ - int fg = pMem->flags; const int nByte = 32; assert( pMem->db==0 || sqlite3_mutex_held(pMem->db->mutex) ); - assert( !(fg&MEM_Zero) ); - assert( !(fg&(MEM_Str|MEM_Blob)) ); - assert( fg&(MEM_Int|MEM_Real) ); + assert( !(pMem->flags&MEM_Zero) ); + assert( !(pMem->flags&(MEM_Str|MEM_Blob)) ); + assert( pMem->flags&(MEM_Int|MEM_Real|MEM_IntReal) ); assert( !sqlite3VdbeMemIsRowSet(pMem) ); assert( EIGHT_BYTE_ALIGNMENT(pMem) );@@ -74018,22 +74907,12 @@ pMem->enc = 0;
return SQLITE_NOMEM_BKPT; } - /* For a Real or Integer, use sqlite3_snprintf() to produce the UTF-8 - ** string representation of the value. Then, if the required encoding - ** is UTF-16le or UTF-16be do a translation. - ** - ** FIX ME: It would be better if sqlite3_snprintf() could do UTF-16. - */ - if( fg & MEM_Int ){ - sqlite3_snprintf(nByte, pMem->z, "%lld", pMem->u.i); - }else{ - assert( fg & MEM_Real ); - sqlite3_snprintf(nByte, pMem->z, "%!.15g", pMem->u.r); - } - pMem->n = sqlite3Strlen30(pMem->z); + vdbeMemRenderNum(nByte, pMem->z, pMem); + assert( pMem->z!=0 ); + pMem->n = sqlite3Strlen30NN(pMem->z); pMem->enc = SQLITE_UTF8; pMem->flags |= MEM_Str|MEM_Term; - if( bForce ) pMem->flags &= ~(MEM_Int|MEM_Real); + if( bForce ) pMem->flags &= ~(MEM_Int|MEM_Real|MEM_IntReal); sqlite3VdbeChangeEncoding(pMem, enc); return SQLITE_OK; }@@ -74207,7 +75086,8 @@ int flags;
assert( pMem->db==0 || sqlite3_mutex_held(pMem->db->mutex) ); assert( EIGHT_BYTE_ALIGNMENT(pMem) ); flags = pMem->flags; - if( flags & MEM_Int ){ + if( flags & (MEM_Int|MEM_IntReal) ){ + testcase( flags & MEM_IntReal ); return pMem->u.i; }else if( flags & MEM_Real ){ return doubleToInt64(pMem->u.r);@@ -74236,7 +75116,8 @@ assert( pMem->db==0 || sqlite3_mutex_held(pMem->db->mutex) );
assert( EIGHT_BYTE_ALIGNMENT(pMem) ); if( pMem->flags & MEM_Real ){ return pMem->u.r; - }else if( pMem->flags & MEM_Int ){ + }else if( pMem->flags & (MEM_Int|MEM_IntReal) ){ + testcase( pMem->flags & MEM_IntReal ); return (double)pMem->u.i; }else if( pMem->flags & (MEM_Str|MEM_Blob) ){ return memRealValue(pMem);@@ -74251,7 +75132,8 @@ ** Return 1 if pMem represents true, and return 0 if pMem represents false.
** Return the value ifNull if pMem is NULL. */ SQLITE_PRIVATE int sqlite3VdbeBooleanValue(Mem *pMem, int ifNull){ - if( pMem->flags & MEM_Int ) return pMem->u.i!=0; + testcase( pMem->flags & MEM_IntReal ); + if( pMem->flags & (MEM_Int|MEM_IntReal) ) return pMem->u.i!=0; if( pMem->flags & MEM_Null ) return ifNull; return sqlite3VdbeRealValue(pMem)!=0.0; }@@ -74314,17 +75196,21 @@
/* Compare a floating point value to an integer. Return true if the two ** values are the same within the precision of the floating point value. ** +** This function assumes that i was obtained by assignment from r1. +** ** For some versions of GCC on 32-bit machines, if you do the more obvious ** comparison of "r1==(double)i" you sometimes get an answer of false even ** though the r1 and (double)i values are bit-for-bit the same. */ -static int sqlite3RealSameAsInt(double r1, sqlite3_int64 i){ +SQLITE_PRIVATE int sqlite3RealSameAsInt(double r1, sqlite3_int64 i){ double r2 = (double)i; - return memcmp(&r1, &r2, sizeof(r1))==0; + return r1==0.0 + || (memcmp(&r1, &r2, sizeof(r1))==0 + && i >= -2251799813685248LL && i < 2251799813685248LL); } /* -** Convert pMem so that it has types MEM_Real or MEM_Int or both. +** Convert pMem so that it has type MEM_Real or MEM_Int. ** Invalidate any prior representations. ** ** Every effort is made to force the conversion, even if the input@@ -74332,25 +75218,26 @@ ** is a string that does not look completely like a number. Convert
** as much of the string as we can and ignore the rest. */ SQLITE_PRIVATE int sqlite3VdbeMemNumerify(Mem *pMem){ - if( (pMem->flags & (MEM_Int|MEM_Real|MEM_Null))==0 ){ + testcase( pMem->flags & MEM_Int ); + testcase( pMem->flags & MEM_Real ); + testcase( pMem->flags & MEM_IntReal ); + testcase( pMem->flags & MEM_Null ); + if( (pMem->flags & (MEM_Int|MEM_Real|MEM_IntReal|MEM_Null))==0 ){ int rc; + sqlite3_int64 ix; assert( (pMem->flags & (MEM_Blob|MEM_Str))!=0 ); assert( pMem->db==0 || sqlite3_mutex_held(pMem->db->mutex) ); - rc = sqlite3Atoi64(pMem->z, &pMem->u.i, pMem->n, pMem->enc); - if( rc==0 ){ + rc = sqlite3AtoF(pMem->z, &pMem->u.r, pMem->n, pMem->enc); + if( ((rc==0 || rc==1) && sqlite3Atoi64(pMem->z, &ix, pMem->n, pMem->enc)<=1) + || sqlite3RealSameAsInt(pMem->u.r, (ix = (i64)pMem->u.r)) + ){ + pMem->u.i = ix; MemSetTypeFlag(pMem, MEM_Int); }else{ - i64 i = pMem->u.i; - sqlite3AtoF(pMem->z, &pMem->u.r, pMem->n, pMem->enc); - if( rc==1 && sqlite3RealSameAsInt(pMem->u.r, i) ){ - pMem->u.i = i; - MemSetTypeFlag(pMem, MEM_Int); - }else{ - MemSetTypeFlag(pMem, MEM_Real); - } + MemSetTypeFlag(pMem, MEM_Real); } } - assert( (pMem->flags & (MEM_Int|MEM_Real|MEM_Null))!=0 ); + assert( (pMem->flags & (MEM_Int|MEM_Real|MEM_IntReal|MEM_Null))!=0 ); pMem->flags &= ~(MEM_Str|MEM_Blob|MEM_Zero); return SQLITE_OK; }@@ -74393,7 +75280,7 @@ assert( MEM_Str==(MEM_Blob>>3) );
pMem->flags |= (pMem->flags&MEM_Blob)>>3; sqlite3ValueApplyAffinity(pMem, SQLITE_AFF_TEXT, encoding); assert( pMem->flags & MEM_Str || pMem->db->mallocFailed ); - pMem->flags &= ~(MEM_Int|MEM_Real|MEM_Blob|MEM_Zero); + pMem->flags &= ~(MEM_Int|MEM_Real|MEM_IntReal|MEM_Blob|MEM_Zero); break; } }@@ -74577,7 +75464,7 @@ ** function for pX. Minor changes, such as adding or removing a
** dual type, are allowed, as long as the underlying value is the ** same. */ u16 mFlags = pMem->flags & pX->flags & pX->mScopyFlags; - assert( (mFlags&MEM_Int)==0 || pMem->u.i==pX->u.i ); + assert( (mFlags&(MEM_Int|MEM_IntReal))==0 || pMem->u.i==pX->u.i ); assert( (mFlags&MEM_Real)==0 || pMem->u.r==pX->u.r ); assert( (mFlags&MEM_Str)==0 || (pMem->n==pX->n && pMem->z==pX->z) ); assert( (mFlags&MEM_Blob)==0 || sqlite3BlobCompare(pMem,pX)==0 );@@ -74699,7 +75586,6 @@ if( nByte<0 ){
assert( enc!=0 ); if( enc==SQLITE_UTF8 ){ nByte = 0x7fffffff & (int)strlen(z); - if( nByte>iLimit ) nByte = iLimit+1; }else{ for(nByte=0; nByte<=iLimit && (z[nByte] | z[nByte+1]); nByte+=2){} }@@ -74711,29 +75597,30 @@ ** also sets a flag in local variable "flags" to indicate the memory
** management (one of MEM_Dyn or MEM_Static). */ if( xDel==SQLITE_TRANSIENT ){ - int nAlloc = nByte; + u32 nAlloc = nByte; if( flags&MEM_Term ){ nAlloc += (enc==SQLITE_UTF8?1:2); } if( nByte>iLimit ){ - return SQLITE_TOOBIG; + return sqlite3ErrorToParser(pMem->db, SQLITE_TOOBIG); } testcase( nAlloc==0 ); testcase( nAlloc==31 ); testcase( nAlloc==32 ); - if( sqlite3VdbeMemClearAndResize(pMem, MAX(nAlloc,32)) ){ + if( sqlite3VdbeMemClearAndResize(pMem, (int)MAX(nAlloc,32)) ){ return SQLITE_NOMEM_BKPT; } memcpy(pMem->z, z, nAlloc); - }else if( xDel==SQLITE_DYNAMIC ){ - sqlite3VdbeMemRelease(pMem); - pMem->zMalloc = pMem->z = (char *)z; - pMem->szMalloc = sqlite3DbMallocSize(pMem->db, pMem->zMalloc); }else{ sqlite3VdbeMemRelease(pMem); pMem->z = (char *)z; - pMem->xDel = xDel; - flags |= ((xDel==SQLITE_STATIC)?MEM_Static:MEM_Dyn); + if( xDel==SQLITE_DYNAMIC ){ + pMem->zMalloc = pMem->z; + pMem->szMalloc = sqlite3DbMallocSize(pMem->db, pMem->zMalloc); + }else{ + pMem->xDel = xDel; + flags |= ((xDel==SQLITE_STATIC)?MEM_Static:MEM_Dyn); + } } pMem->n = nByte;@@ -74776,6 +75663,9 @@ Mem *pMem /* OUT: Return data in this Mem structure. */
){ int rc; pMem->flags = MEM_Null; + if( sqlite3BtreeMaxRecordSize(pCur)<offset+amt ){ + return SQLITE_CORRUPT_BKPT; + } if( SQLITE_OK==(rc = sqlite3VdbeMemClearAndResize(pMem, amt+1)) ){ rc = sqlite3BtreePayload(pCur, offset, amt, pMem->z); if( rc==SQLITE_OK ){@@ -74849,7 +75739,7 @@ }
assert(pVal->enc==(enc & ~SQLITE_UTF16_ALIGNED) || pVal->db==0 || pVal->db->mallocFailed ); if( pVal->enc==(enc & ~SQLITE_UTF16_ALIGNED) ){ - assert( sqlite3VdbeMemConsistentDualRep(pVal) ); + assert( sqlite3VdbeMemValidStrRep(pVal) ); return pVal->z; }else{ return 0;@@ -74872,7 +75762,7 @@ assert( pVal->db==0 || sqlite3_mutex_held(pVal->db->mutex) );
assert( (enc&3)==(enc&~SQLITE_UTF16_ALIGNED) ); assert( !sqlite3VdbeMemIsRowSet(pVal) ); if( (pVal->flags&(MEM_Str|MEM_Term))==(MEM_Str|MEM_Term) && pVal->enc==enc ){ - assert( sqlite3VdbeMemConsistentDualRep(pVal) ); + assert( sqlite3VdbeMemValidStrRep(pVal) ); return pVal->z; } if( pVal->flags&MEM_Null ){@@ -75137,7 +76027,12 @@ sqlite3ValueApplyAffinity(pVal, SQLITE_AFF_NUMERIC, SQLITE_UTF8);
}else{ sqlite3ValueApplyAffinity(pVal, affinity, SQLITE_UTF8); } - if( pVal->flags & (MEM_Int|MEM_Real) ) pVal->flags &= ~MEM_Str; + assert( (pVal->flags & MEM_IntReal)==0 ); + if( pVal->flags & (MEM_Int|MEM_IntReal|MEM_Real) ){ + testcase( pVal->flags & MEM_Int ); + testcase( pVal->flags & MEM_Real ); + pVal->flags &= ~MEM_Str; + } if( enc!=SQLITE_UTF8 ){ rc = sqlite3VdbeChangeEncoding(pVal, enc); }@@ -75160,7 +76055,7 @@ }
}else if( op==TK_NULL ){ pVal = valueNew(db, pCtx); if( pVal==0 ) goto no_mem; - sqlite3VdbeMemNumerify(pVal); + sqlite3VdbeMemSetNull(pVal); } #ifndef SQLITE_OMIT_BLOB_LITERAL else if( op==TK_BLOB ){@@ -75182,9 +76077,11 @@ rc = valueFromFunction(db, pExpr, enc, affinity, &pVal, pCtx);
} #endif else if( op==TK_TRUEFALSE ){ - pVal = valueNew(db, pCtx); - pVal->flags = MEM_Int; - pVal->u.i = pExpr->u.zToken[4]==0; + pVal = valueNew(db, pCtx); + if( pVal ){ + pVal->flags = MEM_Int; + pVal->u.i = pExpr->u.zToken[4]==0; + } } *ppVal = pVal;@@ -75577,7 +76474,7 @@ p->pParse = pParse;
pParse->pVdbe = p; assert( pParse->aLabel==0 ); assert( pParse->nLabel==0 ); - assert( pParse->nOpAlloc==0 ); + assert( p->nOpAlloc==0 ); assert( pParse->szOpAlloc==0 ); sqlite3VdbeAddOp2(p, OP_Init, 0, 1); return p;@@ -75607,6 +76504,43 @@ assert( p->zSql==0 );
p->zSql = sqlite3DbStrNDup(p->db, z, n); } +#ifdef SQLITE_ENABLE_NORMALIZE +/* +** Add a new element to the Vdbe->pDblStr list. +*/ +SQLITE_PRIVATE void sqlite3VdbeAddDblquoteStr(sqlite3 *db, Vdbe *p, const char *z){ + if( p ){ + int n = sqlite3Strlen30(z); + DblquoteStr *pStr = sqlite3DbMallocRawNN(db, + sizeof(*pStr)+n+1-sizeof(pStr->z)); + if( pStr ){ + pStr->pNextStr = p->pDblStr; + p->pDblStr = pStr; + memcpy(pStr->z, z, n+1); + } + } +} +#endif + +#ifdef SQLITE_ENABLE_NORMALIZE +/* +** zId of length nId is a double-quoted identifier. Check to see if +** that identifier is really used as a string literal. +*/ +SQLITE_PRIVATE int sqlite3VdbeUsesDoubleQuotedString( + Vdbe *pVdbe, /* The prepared statement */ + const char *zId /* The double-quoted identifier, already dequoted */ +){ + DblquoteStr *pStr; + assert( zId!=0 ); + if( pVdbe->pDblStr==0 ) return 0; + for(pStr=pVdbe->pDblStr; pStr; pStr=pStr->pNextStr){ + if( strcmp(zId, pStr->z)==0 ) return 1; + } + return 0; +} +#endif + /* ** Swap all content between two VDBE structures. */@@ -75626,6 +76560,11 @@ pB->pPrev = pTmp;
zTmp = pA->zSql; pA->zSql = pB->zSql; pB->zSql = zTmp; +#if 0 + zTmp = pA->zNormSql; + pA->zNormSql = pB->zNormSql; + pB->zNormSql = zTmp; +#endif pB->expmask = pA->expmask; pB->prepFlags = pA->prepFlags; memcpy(pB->aCounter, pA->aCounter, sizeof(pB->aCounter));@@ -75638,7 +76577,7 @@ ** than its current size. nOp is guaranteed to be less than or equal
** to 1024/sizeof(Op). ** ** If an out-of-memory error occurs while resizing the array, return -** SQLITE_NOMEM. In this case Vdbe.aOp and Parse.nOpAlloc remain +** SQLITE_NOMEM. In this case Vdbe.aOp and Vdbe.nOpAlloc remain ** unchanged (this is so that any opcodes already allocated can be ** correctly deallocated along with the rest of the Vdbe). */@@ -75654,9 +76593,11 @@ ** by the minimum* amount required until the size reaches 512. Normal
** operation (without SQLITE_TEST_REALLOC_STRESS) is to double the current ** size of the op array or add 1KB of space, whichever is smaller. */ #ifdef SQLITE_TEST_REALLOC_STRESS - int nNew = (p->nOpAlloc>=512 ? p->nOpAlloc*2 : p->nOpAlloc+nOp); + sqlite3_int64 nNew = (v->nOpAlloc>=512 ? 2*(sqlite3_int64)v->nOpAlloc + : (sqlite3_int64)v->nOpAlloc+nOp); #else - int nNew = (p->nOpAlloc ? p->nOpAlloc*2 : (int)(1024/sizeof(Op))); + sqlite3_int64 nNew = (v->nOpAlloc ? 2*(sqlite3_int64)v->nOpAlloc + : (sqlite3_int64)(1024/sizeof(Op))); UNUSED_PARAMETER(nOp); #endif@@ -75667,11 +76608,11 @@ return SQLITE_NOMEM;
} assert( nOp<=(1024/sizeof(Op)) ); - assert( nNew>=(p->nOpAlloc+nOp) ); + assert( nNew>=(v->nOpAlloc+nOp) ); pNew = sqlite3DbRealloc(p->db, v->aOp, nNew*sizeof(Op)); if( pNew ){ p->szOpAlloc = sqlite3DbMallocSize(p->db, pNew); - p->nOpAlloc = p->szOpAlloc/sizeof(Op); + v->nOpAlloc = p->szOpAlloc/sizeof(Op); v->aOp = pNew; } return (pNew ? SQLITE_OK : SQLITE_NOMEM_BKPT);@@ -75705,9 +76646,9 @@ ** the sqlite3VdbeChangeP4() function to change the value of the P4
** operand. */ static SQLITE_NOINLINE int growOp3(Vdbe *p, int op, int p1, int p2, int p3){ - assert( p->pParse->nOpAlloc<=p->nOp ); + assert( p->nOpAlloc<=p->nOp ); if( growOpArray(p, 1) ) return 1; - assert( p->pParse->nOpAlloc>p->nOp ); + assert( p->nOpAlloc>p->nOp ); return sqlite3VdbeAddOp3(p, op, p1, p2, p3); } SQLITE_PRIVATE int sqlite3VdbeAddOp3(Vdbe *p, int op, int p1, int p2, int p3){@@ -75717,7 +76658,7 @@
i = p->nOp; assert( p->magic==VDBE_MAGIC_INIT ); assert( op>=0 && op<0xff ); - if( p->pParse->nOpAlloc<=i ){ + if( p->nOpAlloc<=i ){ return growOp3(p, op, p1, p2, p3); } p->nOp++;@@ -75849,13 +76790,29 @@ return pOp->p2;
} /* -** Add a new OP_Explain opcode. +** Set a debugger breakpoint on the following routine in order to +** monitor the EXPLAIN QUERY PLAN code generation. +*/ +#if defined(SQLITE_DEBUG) +SQLITE_PRIVATE void sqlite3ExplainBreakpoint(const char *z1, const char *z2){ + (void)z1; + (void)z2; +} +#endif + +/* +** Add a new OP_ opcode. ** ** If the bPush flag is true, then make this opcode the parent for ** subsequent Explains until sqlite3VdbeExplainPop() is called. */ SQLITE_PRIVATE void sqlite3VdbeExplain(Parse *pParse, u8 bPush, const char *zFmt, ...){ - if( pParse->explain==2 ){ +#ifndef SQLITE_DEBUG + /* Always include the OP_Explain opcodes if SQLITE_DEBUG is defined. + ** But omit them (for performance) during production builds */ + if( pParse->explain==2 ) +#endif + { char *zMsg; Vdbe *v; va_list ap;@@ -75867,7 +76824,10 @@ v = pParse->pVdbe;
iThis = v->nOp; sqlite3VdbeAddOp4(v, OP_Explain, iThis, pParse->addrExplain, 0, zMsg, P4_DYNAMIC); - if( bPush) pParse->addrExplain = iThis; + sqlite3ExplainBreakpoint(bPush?"PUSH":"", sqlite3VdbeGetOp(v,-1)->p4.z); + if( bPush){ + pParse->addrExplain = iThis; + } } }@@ -75875,6 +76835,7 @@ /*
** Pop the EXPLAIN QUERY PLAN stack one level. */ SQLITE_PRIVATE void sqlite3VdbeExplainPop(Parse *pParse){ + sqlite3ExplainBreakpoint("POP", 0); pParse->addrExplain = sqlite3VdbeExplainParent(pParse); } #endif /* SQLITE_OMIT_EXPLAIN */@@ -75939,21 +76900,22 @@ **
** The VDBE knows that a P2 value is a label because labels are ** always negative and P2 values are suppose to be non-negative. ** Hence, a negative P2 value is a label that has yet to be resolved. +** (Later:) This is only true for opcodes that have the OPFLG_JUMP +** property. ** -** Zero is returned if a malloc() fails. +** Variable usage notes: +** +** Parse.aLabel[x] Stores the address that the x-th label resolves +** into. For testing (SQLITE_DEBUG), unresolved +** labels stores -1, but that is not required. +** Parse.nLabelAlloc Number of slots allocated to Parse.aLabel[] +** Parse.nLabel The *negative* of the number of labels that have +** been issued. The negative is stored because +** that gives a performance improvement over storing +** the equivalent positive value. */ -SQLITE_PRIVATE int sqlite3VdbeMakeLabel(Vdbe *v){ - Parse *p = v->pParse; - int i = p->nLabel++; - assert( v->magic==VDBE_MAGIC_INIT ); - if( (i & (i-1))==0 ){ - p->aLabel = sqlite3DbReallocOrFree(p->db, p->aLabel, - (i*2+1)*sizeof(p->aLabel[0])); - } - if( p->aLabel ){ - p->aLabel[i] = -1; - } - return ADDR(i); +SQLITE_PRIVATE int sqlite3VdbeMakeLabel(Parse *pParse){ + return --pParse->nLabel; } /*@@ -75961,18 +76923,35 @@ ** Resolve label "x" to be the address of the next instruction to
** be inserted. The parameter "x" must have been obtained from ** a prior call to sqlite3VdbeMakeLabel(). */ +static SQLITE_NOINLINE void resizeResolveLabel(Parse *p, Vdbe *v, int j){ + int nNewSize = 10 - p->nLabel; + p->aLabel = sqlite3DbReallocOrFree(p->db, p->aLabel, + nNewSize*sizeof(p->aLabel[0])); + if( p->aLabel==0 ){ + p->nLabelAlloc = 0; + }else{ +#ifdef SQLITE_DEBUG + int i; + for(i=p->nLabelAlloc; i<nNewSize; i++) p->aLabel[i] = -1; +#endif + p->nLabelAlloc = nNewSize; + p->aLabel[j] = v->nOp; + } +} SQLITE_PRIVATE void sqlite3VdbeResolveLabel(Vdbe *v, int x){ Parse *p = v->pParse; int j = ADDR(x); assert( v->magic==VDBE_MAGIC_INIT ); - assert( j<p->nLabel ); + assert( j<-p->nLabel ); assert( j>=0 ); - if( p->aLabel ){ #ifdef SQLITE_DEBUG - if( p->db->flags & SQLITE_VdbeAddopTrace ){ - printf("RESOLVE LABEL %d to %d\n", x, v->nOp); - } + if( p->db->flags & SQLITE_VdbeAddopTrace ){ + printf("RESOLVE LABEL %d to %d\n", x, v->nOp); + } #endif + if( p->nLabelAlloc + p->nLabel < 0 ){ + resizeResolveLabel(p,v,j); + }else{ assert( p->aLabel[j]==(-1) ); /* Labels may only be resolved once */ p->aLabel[j] = v->nOp; }@@ -76088,6 +77067,7 @@ SQLITE_PRIVATE int sqlite3VdbeAssertMayAbort(Vdbe *v, int mayAbort){
int hasAbort = 0; int hasFkCounter = 0; int hasCreateTable = 0; + int hasCreateIndex = 0; int hasInitCoroutine = 0; Op *pOp; VdbeOpIter sIter;@@ -76097,13 +77077,23 @@
while( (pOp = opIterNext(&sIter))!=0 ){ int opcode = pOp->opcode; if( opcode==OP_Destroy || opcode==OP_VUpdate || opcode==OP_VRename + || opcode==OP_VDestroy + || (opcode==OP_Function0 && pOp->p4.pFunc->funcFlags&SQLITE_FUNC_INTERNAL) || ((opcode==OP_Halt || opcode==OP_HaltIfNull) - && ((pOp->p1&0xff)==SQLITE_CONSTRAINT && pOp->p2==OE_Abort)) + && ((pOp->p1)!=SQLITE_OK && pOp->p2==OE_Abort)) ){ hasAbort = 1; break; } if( opcode==OP_CreateBtree && pOp->p3==BTREE_INTKEY ) hasCreateTable = 1; + if( mayAbort ){ + /* hasCreateIndex may also be set for some DELETE statements that use + ** OP_Clear. So this routine may end up returning true in the case + ** where a "DELETE FROM tbl" has a statement-journal but does not + ** require one. This is not so bad - it is an inefficiency, not a bug. */ + if( opcode==OP_CreateBtree && pOp->p3==BTREE_BLOBKEY ) hasCreateIndex = 1; + if( opcode==OP_Clear ) hasCreateIndex = 1; + } if( opcode==OP_InitCoroutine ) hasInitCoroutine = 1; #ifndef SQLITE_OMIT_FOREIGN_KEY if( opcode==OP_FkCounter && pOp->p1==0 && pOp->p2==1 ){@@ -76119,7 +77109,8 @@ ** through all opcodes and hasAbort may be set incorrectly. Return
** true for this case to prevent the assert() in the callers frame ** from failing. */ return ( v->db->mallocFailed || hasAbort==mayAbort || hasFkCounter - || (hasCreateTable && hasInitCoroutine) ); + || (hasCreateTable && hasInitCoroutine) || hasCreateIndex + ); } #endif /* SQLITE_DEBUG - the sqlite3AssertMayAbort() function */@@ -76247,7 +77238,7 @@ /* The mkopcodeh.tcl script has so arranged things that the only
** non-jump opcodes less than SQLITE_MX_JUMP_CODE are guaranteed to ** have non-negative values for P2. */ assert( (sqlite3OpcodeProperty[pOp->opcode] & OPFLG_JUMP)!=0 ); - assert( ADDR(pOp->p2)<pParse->nLabel ); + assert( ADDR(pOp->p2)<-pParse->nLabel ); pOp->p2 = aLabel[ADDR(pOp->p2)]; } break;@@ -76286,7 +77277,7 @@ ** sqlite3VdbeAddOpList() will always be non-NULL.
*/ #if defined(SQLITE_DEBUG) && !defined(SQLITE_TEST_REALLOC_STRESS) SQLITE_PRIVATE void sqlite3VdbeVerifyNoMallocRequired(Vdbe *p, int N){ - assert( p->nOp + N <= p->pParse->nOpAlloc ); + assert( p->nOp + N <= p->nOpAlloc ); } #endif@@ -76358,7 +77349,7 @@ int i;
VdbeOp *pOut, *pFirst; assert( nOp>0 ); assert( p->magic==VDBE_MAGIC_INIT ); - if( p->nOp + nOp > p->pParse->nOpAlloc && growOpArray(p, nOp) ){ + if( p->nOp + nOp > p->nOpAlloc && growOpArray(p, nOp) ){ return 0; } pFirst = pOut = &p->aOp[p->nOp];@@ -76404,7 +77395,7 @@ int addrVisit, /* Address of rows visited counter */
LogEst nEst, /* Estimated number of output rows */ const char *zName /* Name of table or index being scanned */ ){ - int nByte = (p->nScan+1) * sizeof(ScanStatus); + sqlite3_int64 nByte = (p->nScan+1) * sizeof(ScanStatus); ScanStatus *aNew; aNew = (ScanStatus*)sqlite3DbRealloc(p->db, p->aScan, nByte); if( aNew ){@@ -76991,7 +77982,7 @@ case P4_MEM: {
Mem *pMem = pOp->p4.pMem; if( pMem->flags & MEM_Str ){ zP4 = pMem->z; - }else if( pMem->flags & MEM_Int ){ + }else if( pMem->flags & (MEM_Int|MEM_IntReal) ){ sqlite3_str_appendf(&x, "%lld", pMem->u.i); }else if( pMem->flags & MEM_Real ){ sqlite3_str_appendf(&x, "%.16g", pMem->u.r);@@ -77525,9 +78516,9 @@ ** by subcomponents of a prepared statement. Space is allocated out
** of a ReusableSpace object by the allocSpace() routine below. */ struct ReusableSpace { - u8 *pSpace; /* Available memory */ - int nFree; /* Bytes of available memory */ - int nNeeded; /* Total bytes that could not be allocated */ + u8 *pSpace; /* Available memory */ + sqlite3_int64 nFree; /* Bytes of available memory */ + sqlite3_int64 nNeeded; /* Total bytes that could not be allocated */ }; /* Try to allocate nByte bytes of 8-byte aligned bulk memory for pBuf@@ -77547,7 +78538,7 @@ */
static void *allocSpace( struct ReusableSpace *p, /* Bulk memory available for allocation */ void *pBuf, /* Pointer to a prior allocation */ - int nByte /* Bytes of memory needed */ + sqlite3_int64 nByte /* Bytes of memory needed */ ){ assert( EIGHT_BYTE_ALIGNMENT(p->pSpace) ); if( pBuf==0 ){@@ -77680,19 +78671,27 @@ ** This two-pass approach that reuses as much memory as possible from
** the leftover memory at the end of the opcode array. This can significantly ** reduce the amount of memory held by a prepared statement. */ - do { - x.nNeeded = 0; - p->aMem = allocSpace(&x, p->aMem, nMem*sizeof(Mem)); - p->aVar = allocSpace(&x, p->aVar, nVar*sizeof(Mem)); - p->apArg = allocSpace(&x, p->apArg, nArg*sizeof(Mem*)); - p->apCsr = allocSpace(&x, p->apCsr, nCursor*sizeof(VdbeCursor*)); + x.nNeeded = 0; + p->aMem = allocSpace(&x, 0, nMem*sizeof(Mem)); + p->aVar = allocSpace(&x, 0, nVar*sizeof(Mem)); + p->apArg = allocSpace(&x, 0, nArg*sizeof(Mem*)); + p->apCsr = allocSpace(&x, 0, nCursor*sizeof(VdbeCursor*)); #ifdef SQLITE_ENABLE_STMT_SCANSTATUS - p->anExec = allocSpace(&x, p->anExec, p->nOp*sizeof(i64)); + p->anExec = allocSpace(&x, 0, p->nOp*sizeof(i64)); #endif - if( x.nNeeded==0 ) break; + if( x.nNeeded ){ x.pSpace = p->pFree = sqlite3DbMallocRawNN(db, x.nNeeded); x.nFree = x.nNeeded; - }while( !db->mallocFailed ); + if( !db->mallocFailed ){ + p->aMem = allocSpace(&x, p->aMem, nMem*sizeof(Mem)); + p->aVar = allocSpace(&x, p->aVar, nVar*sizeof(Mem)); + p->apArg = allocSpace(&x, p->apArg, nArg*sizeof(Mem*)); + p->apCsr = allocSpace(&x, p->apCsr, nCursor*sizeof(VdbeCursor*)); +#ifdef SQLITE_ENABLE_STMT_SCANSTATUS + p->anExec = allocSpace(&x, p->anExec, p->nOp*sizeof(i64)); +#endif + } + } p->pVList = pParse->pVList; pParse->pVList = 0;@@ -78345,7 +79344,7 @@ }
} /* Check for immediate foreign key violations. */ - if( p->rc==SQLITE_OK ){ + if( p->rc==SQLITE_OK || (p->errorAction==OE_Fail && !isSpecialError) ){ sqlite3VdbeCheckFk(p, 0); }@@ -78384,7 +79383,7 @@ p->nChange = 0;
}else{ db->nDeferredCons = 0; db->nDeferredImmCons = 0; - db->flags &= ~SQLITE_DeferFKs; + db->flags &= ~(u64)SQLITE_DeferFKs; sqlite3CommitInternalChanges(db); } }else{@@ -78697,6 +79696,16 @@ }
vdbeFreeOpArray(db, p->aOp, p->nOp); sqlite3DbFree(db, p->aColName); sqlite3DbFree(db, p->zSql); +#ifdef SQLITE_ENABLE_NORMALIZE + sqlite3DbFree(db, p->zNormSql); + { + DblquoteStr *pThis, *pNext; + for(pThis=p->pDblStr; pThis; pThis=pNext){ + pNext = pThis->pNextStr; + sqlite3DbFree(db, pThis); + } + } +#endif #ifdef SQLITE_ENABLE_STMT_SCANSTATUS { int i;@@ -78861,6 +79870,8 @@ */
/* ** Return the serial-type for the value stored in pMem. +** +** This routine might convert a large MEM_IntReal value into MEM_Real. */ SQLITE_PRIVATE u32 sqlite3VdbeSerialType(Mem *pMem, int file_format, u32 *pLen){ int flags = pMem->flags;@@ -78871,11 +79882,13 @@ if( flags&MEM_Null ){
*pLen = 0; return 0; } - if( flags&MEM_Int ){ + if( flags&(MEM_Int|MEM_IntReal) ){ /* Figure out whether to use 1, 2, 4, 6 or 8 bytes. */ # define MAX_6BYTE ((((i64)0x00008000)<<32)-1) i64 i = pMem->u.i; u64 u; + testcase( flags & MEM_Int ); + testcase( flags & MEM_IntReal ); if( i<0 ){ u = ~i; }else{@@ -78895,6 +79908,15 @@ if( u<=8388607 ){ *pLen = 3; return 3; }
if( u<=2147483647 ){ *pLen = 4; return 4; } if( u<=MAX_6BYTE ){ *pLen = 6; return 5; } *pLen = 8; + if( flags&MEM_IntReal ){ + /* If the value is IntReal and is going to take up 8 bytes to store + ** as an integer, then we might as well make it an 8-byte floating + ** point value */ + pMem->u.r = (double)pMem->u.i; + pMem->flags &= ~MEM_IntReal; + pMem->flags |= MEM_Real; + return 7; + } return 6; } if( flags&MEM_Real ){@@ -79068,7 +80090,7 @@ ** The few cases that require local variables are broken out into a separate
** routine so that in most cases the overhead of moving the stack pointer ** is avoided. */ -static u32 SQLITE_NOINLINE serialGet( +static u32 serialGet( const unsigned char *buf, /* Buffer to deserialize from */ u32 serial_type, /* Serial type to deserialize */ Mem *pMem /* Memory cell to write value into */@@ -79100,7 +80122,7 @@ #endif
assert( sizeof(x)==8 && sizeof(pMem->u.r)==8 ); swapMixedEndianFloat(x); memcpy(&pMem->u.r, &x, sizeof(x)); - pMem->flags = sqlite3IsNaN(pMem->u.r) ? MEM_Null : MEM_Real; + pMem->flags = IsNaN(x) ? MEM_Null : MEM_Real; } return 8; }@@ -79236,7 +80258,7 @@ const void *pKey, /* The binary record */
UnpackedRecord *p /* Populate this structure before returning. */ ){ const unsigned char *aKey = (const unsigned char *)pKey; - int d; + u32 d; u32 idx; /* Offset in aKey[] to read from */ u16 u; /* Unsigned loop counter */ u32 szHdr;@@ -79247,7 +80269,7 @@ assert( EIGHT_BYTE_ALIGNMENT(pMem) );
idx = getVarint32(aKey, szHdr); d = szHdr; u = 0; - while( idx<szHdr && d<=nKey ){ + while( idx<szHdr && d<=(u32)nKey ){ u32 serial_type; idx += getVarint32(&aKey[idx], serial_type);@@ -79260,6 +80282,13 @@ d += sqlite3VdbeSerialGet(&aKey[d], serial_type, pMem);
pMem++; if( (++u)>=p->nField ) break; } + if( d>(u32)nKey && u ){ + assert( CORRUPT_DB ); + /* In a corrupt record entry, the last pMem might have been set up using + ** uninitialized memory. Overwrite its value with NULL, to prevent + ** warnings from MSAN. */ + sqlite3VdbeMemSetNull(pMem-1); + } assert( u<=pKeyInfo->nKeyField + 1 ); p->nField = u; }@@ -79325,8 +80354,8 @@ ** always be greater than or equal to the amount of required key space.
** Use that approximation to avoid the more expensive call to ** sqlite3VdbeSerialTypeLen() in the common case. */ - if( d1+serial_type1+2>(u32)nKey1 - && d1+sqlite3VdbeSerialTypeLen(serial_type1)>(u32)nKey1 + if( d1+(u64)serial_type1+2>(u64)nKey1 + && d1+(u64)sqlite3VdbeSerialTypeLen(serial_type1)>(u64)nKey1 ){ break; }@@ -79337,7 +80366,8 @@ d1 += sqlite3VdbeSerialGet(&aKey1[d1], serial_type1, &mem1);
/* Do the comparison */ - rc = sqlite3MemCompare(&mem1, &pPKey2->aMem[i], pKeyInfo->aColl[i]); + rc = sqlite3MemCompare(&mem1, &pPKey2->aMem[i], + pKeyInfo->nAllField>i ? pKeyInfo->aColl[i] : 0); if( rc!=0 ){ assert( mem1.szMalloc==0 ); /* See comment below */ if( pKeyInfo->aSortOrder[i] ){@@ -79542,8 +80572,13 @@ }
/* At least one of the two values is a number */ - if( combined_flags&(MEM_Int|MEM_Real) ){ - if( (f1 & f2 & MEM_Int)!=0 ){ + if( combined_flags&(MEM_Int|MEM_Real|MEM_IntReal) ){ + testcase( combined_flags & MEM_Int ); + testcase( combined_flags & MEM_Real ); + testcase( combined_flags & MEM_IntReal ); + if( (f1 & f2 & (MEM_Int|MEM_IntReal))!=0 ){ + testcase( f1 & f2 & MEM_Int ); + testcase( f1 & f2 & MEM_IntReal ); if( pMem1->u.i < pMem2->u.i ) return -1; if( pMem1->u.i > pMem2->u.i ) return +1; return 0;@@ -79553,15 +80588,23 @@ if( pMem1->u.r < pMem2->u.r ) return -1;
if( pMem1->u.r > pMem2->u.r ) return +1; return 0; } - if( (f1&MEM_Int)!=0 ){ + if( (f1&(MEM_Int|MEM_IntReal))!=0 ){ + testcase( f1 & MEM_Int ); + testcase( f1 & MEM_IntReal ); if( (f2&MEM_Real)!=0 ){ return sqlite3IntFloatCompare(pMem1->u.i, pMem2->u.r); + }else if( (f2&(MEM_Int|MEM_IntReal))!=0 ){ + if( pMem1->u.i < pMem2->u.i ) return -1; + if( pMem1->u.i > pMem2->u.i ) return +1; + return 0; }else{ return -1; } } if( (f1&MEM_Real)!=0 ){ - if( (f2&MEM_Int)!=0 ){ + if( (f2&(MEM_Int|MEM_IntReal))!=0 ){ + testcase( f2 & MEM_Int ); + testcase( f2 & MEM_IntReal ); return -sqlite3IntFloatCompare(pMem2->u.i, pMem1->u.r); }else{ return -1;@@ -79693,11 +80736,11 @@ pRhs++;
}else{ idx1 = getVarint32(aKey1, szHdr1); d1 = szHdr1; - if( d1>(unsigned)nKey1 ){ - pPKey2->errCode = (u8)SQLITE_CORRUPT_BKPT; - return 0; /* Corruption */ - } i = 0; + } + if( d1>(unsigned)nKey1 ){ + pPKey2->errCode = (u8)SQLITE_CORRUPT_BKPT; + return 0; /* Corruption */ } VVA_ONLY( mem1.szMalloc = 0; ) /* Only needed by assert() statements */@@ -79710,7 +80753,9 @@ do{
u32 serial_type; /* RHS is an integer */ - if( pRhs->flags & MEM_Int ){ + if( pRhs->flags & (MEM_Int|MEM_IntReal) ){ + testcase( pRhs->flags & MEM_Int ); + testcase( pRhs->flags & MEM_IntReal ); serial_type = aKey1[idx1]; testcase( serial_type==12 ); if( serial_type>=10 ){@@ -79768,10 +80813,12 @@ }else{
mem1.n = (serial_type - 12) / 2; testcase( (d1+mem1.n)==(unsigned)nKey1 ); testcase( (d1+mem1.n+1)==(unsigned)nKey1 ); - if( (d1+mem1.n) > (unsigned)nKey1 ){ + if( (d1+mem1.n) > (unsigned)nKey1 + || (pKeyInfo = pPKey2->pKeyInfo)->nAllField<=i + ){ pPKey2->errCode = (u8)SQLITE_CORRUPT_BKPT; return 0; /* Corruption */ - }else if( (pKeyInfo = pPKey2->pKeyInfo)->aColl[i] ){ + }else if( pKeyInfo->aColl[i] ){ mem1.enc = pKeyInfo->enc; mem1.db = pKeyInfo->db; mem1.flags = MEM_Str;@@ -80053,7 +81100,9 @@ }
testcase( flags & MEM_Real ); testcase( flags & MEM_Null ); testcase( flags & MEM_Blob ); - if( (flags & (MEM_Real|MEM_Null|MEM_Blob))==0 && p->pKeyInfo->aColl[0]==0 ){ + if( (flags & (MEM_Real|MEM_IntReal|MEM_Null|MEM_Blob))==0 + && p->pKeyInfo->aColl[0]==0 + ){ assert( flags & MEM_Str ); return vdbeRecordCompareString; }@@ -80098,7 +81147,9 @@ /* The index entry must begin with a header size */
(void)getVarint32((u8*)m.z, szHdr); testcase( szHdr==3 ); testcase( szHdr==m.n ); - if( unlikely(szHdr<3 || (int)szHdr>m.n) ){ + testcase( szHdr>0x7fffffff ); + assert( m.n>=0 ); + if( unlikely(szHdr<3 || szHdr>(unsigned)m.n) ){ goto idx_rowid_corruption; }@@ -80469,14 +81520,16 @@ static SQLITE_NOINLINE void invokeProfileCallback(sqlite3 *db, Vdbe *p){
sqlite3_int64 iNow; sqlite3_int64 iElapse; assert( p->startTime>0 ); - assert( db->xProfile!=0 || (db->mTrace & SQLITE_TRACE_PROFILE)!=0 ); + assert( (db->mTrace & (SQLITE_TRACE_PROFILE|SQLITE_TRACE_XPROFILE))!=0 ); assert( db->init.busy==0 ); assert( p->zSql!=0 ); sqlite3OsCurrentTimeInt64(db->pVfs, &iNow); iElapse = (iNow - p->startTime)*1000000; +#ifndef SQLITE_OMIT_DEPRECATED if( db->xProfile ){ db->xProfile(db->pProfileArg, p->zSql, iElapse); } +#endif if( db->mTrace & SQLITE_TRACE_PROFILE ){ db->xTrace(SQLITE_TRACE_PROFILE, db->pTraceArg, p, (void*)&iElapse); }@@ -80639,45 +81692,97 @@ ** point number string BLOB NULL
*/ SQLITE_API int sqlite3_value_type(sqlite3_value* pVal){ static const u8 aType[] = { - SQLITE_BLOB, /* 0x00 */ - SQLITE_NULL, /* 0x01 */ - SQLITE_TEXT, /* 0x02 */ - SQLITE_NULL, /* 0x03 */ - SQLITE_INTEGER, /* 0x04 */ - SQLITE_NULL, /* 0x05 */ - SQLITE_INTEGER, /* 0x06 */ - SQLITE_NULL, /* 0x07 */ - SQLITE_FLOAT, /* 0x08 */ - SQLITE_NULL, /* 0x09 */ - SQLITE_FLOAT, /* 0x0a */ - SQLITE_NULL, /* 0x0b */ - SQLITE_INTEGER, /* 0x0c */ - SQLITE_NULL, /* 0x0d */ - SQLITE_INTEGER, /* 0x0e */ - SQLITE_NULL, /* 0x0f */ - SQLITE_BLOB, /* 0x10 */ - SQLITE_NULL, /* 0x11 */ - SQLITE_TEXT, /* 0x12 */ - SQLITE_NULL, /* 0x13 */ - SQLITE_INTEGER, /* 0x14 */ - SQLITE_NULL, /* 0x15 */ - SQLITE_INTEGER, /* 0x16 */ - SQLITE_NULL, /* 0x17 */ - SQLITE_FLOAT, /* 0x18 */ - SQLITE_NULL, /* 0x19 */ - SQLITE_FLOAT, /* 0x1a */ - SQLITE_NULL, /* 0x1b */ - SQLITE_INTEGER, /* 0x1c */ - SQLITE_NULL, /* 0x1d */ - SQLITE_INTEGER, /* 0x1e */ - SQLITE_NULL, /* 0x1f */ + SQLITE_BLOB, /* 0x00 (not possible) */ + SQLITE_NULL, /* 0x01 NULL */ + SQLITE_TEXT, /* 0x02 TEXT */ + SQLITE_NULL, /* 0x03 (not possible) */ + SQLITE_INTEGER, /* 0x04 INTEGER */ + SQLITE_NULL, /* 0x05 (not possible) */ + SQLITE_INTEGER, /* 0x06 INTEGER + TEXT */ + SQLITE_NULL, /* 0x07 (not possible) */ + SQLITE_FLOAT, /* 0x08 FLOAT */ + SQLITE_NULL, /* 0x09 (not possible) */ + SQLITE_FLOAT, /* 0x0a FLOAT + TEXT */ + SQLITE_NULL, /* 0x0b (not possible) */ + SQLITE_INTEGER, /* 0x0c (not possible) */ + SQLITE_NULL, /* 0x0d (not possible) */ + SQLITE_INTEGER, /* 0x0e (not possible) */ + SQLITE_NULL, /* 0x0f (not possible) */ + SQLITE_BLOB, /* 0x10 BLOB */ + SQLITE_NULL, /* 0x11 (not possible) */ + SQLITE_TEXT, /* 0x12 (not possible) */ + SQLITE_NULL, /* 0x13 (not possible) */ + SQLITE_INTEGER, /* 0x14 INTEGER + BLOB */ + SQLITE_NULL, /* 0x15 (not possible) */ + SQLITE_INTEGER, /* 0x16 (not possible) */ + SQLITE_NULL, /* 0x17 (not possible) */ + SQLITE_FLOAT, /* 0x18 FLOAT + BLOB */ + SQLITE_NULL, /* 0x19 (not possible) */ + SQLITE_FLOAT, /* 0x1a (not possible) */ + SQLITE_NULL, /* 0x1b (not possible) */ + SQLITE_INTEGER, /* 0x1c (not possible) */ + SQLITE_NULL, /* 0x1d (not possible) */ + SQLITE_INTEGER, /* 0x1e (not possible) */ + SQLITE_NULL, /* 0x1f (not possible) */ + SQLITE_FLOAT, /* 0x20 INTREAL */ + SQLITE_NULL, /* 0x21 (not possible) */ + SQLITE_TEXT, /* 0x22 INTREAL + TEXT */ + SQLITE_NULL, /* 0x23 (not possible) */ + SQLITE_FLOAT, /* 0x24 (not possible) */ + SQLITE_NULL, /* 0x25 (not possible) */ + SQLITE_FLOAT, /* 0x26 (not possible) */ + SQLITE_NULL, /* 0x27 (not possible) */ + SQLITE_FLOAT, /* 0x28 (not possible) */ + SQLITE_NULL, /* 0x29 (not possible) */ + SQLITE_FLOAT, /* 0x2a (not possible) */ + SQLITE_NULL, /* 0x2b (not possible) */ + SQLITE_FLOAT, /* 0x2c (not possible) */ + SQLITE_NULL, /* 0x2d (not possible) */ + SQLITE_FLOAT, /* 0x2e (not possible) */ + SQLITE_NULL, /* 0x2f (not possible) */ + SQLITE_BLOB, /* 0x30 (not possible) */ + SQLITE_NULL, /* 0x31 (not possible) */ + SQLITE_TEXT, /* 0x32 (not possible) */ + SQLITE_NULL, /* 0x33 (not possible) */ + SQLITE_FLOAT, /* 0x34 (not possible) */ + SQLITE_NULL, /* 0x35 (not possible) */ + SQLITE_FLOAT, /* 0x36 (not possible) */ + SQLITE_NULL, /* 0x37 (not possible) */ + SQLITE_FLOAT, /* 0x38 (not possible) */ + SQLITE_NULL, /* 0x39 (not possible) */ + SQLITE_FLOAT, /* 0x3a (not possible) */ + SQLITE_NULL, /* 0x3b (not possible) */ + SQLITE_FLOAT, /* 0x3c (not possible) */ + SQLITE_NULL, /* 0x3d (not possible) */ + SQLITE_FLOAT, /* 0x3e (not possible) */ + SQLITE_NULL, /* 0x3f (not possible) */ }; +#ifdef SQLITE_DEBUG + { + int eType = SQLITE_BLOB; + if( pVal->flags & MEM_Null ){ + eType = SQLITE_NULL; + }else if( pVal->flags & (MEM_Real|MEM_IntReal) ){ + eType = SQLITE_FLOAT; + }else if( pVal->flags & MEM_Int ){ + eType = SQLITE_INTEGER; + }else if( pVal->flags & MEM_Str ){ + eType = SQLITE_TEXT; + } + assert( eType == aType[pVal->flags&MEM_AffMask] ); + } +#endif return aType[pVal->flags&MEM_AffMask]; } /* Return true if a parameter to xUpdate represents an unchanged column */ SQLITE_API int sqlite3_value_nochange(sqlite3_value *pVal){ return (pVal->flags&(MEM_Null|MEM_Zero))==(MEM_Null|MEM_Zero); +} + +/* Return true if a parameter value originated from an sqlite3_bind() */ +SQLITE_API int sqlite3_value_frombind(sqlite3_value *pVal){ + return (pVal->flags&MEM_FromBind)!=0; } /* Make a copy of an sqlite3_value object@@ -80916,6 +82021,21 @@ pCtx->isError = SQLITE_NOMEM_BKPT;
sqlite3OomFault(pCtx->pOut->db); } +#ifndef SQLITE_UNTESTABLE +/* Force the INT64 value currently stored as the result to be +** a MEM_IntReal value. See the SQLITE_TESTCTRL_RESULT_INTREAL +** test-control. +*/ +SQLITE_PRIVATE void sqlite3ResultIntReal(sqlite3_context *pCtx){ + assert( sqlite3_mutex_held(pCtx->pOut->db->mutex) ); + if( pCtx->pOut->flags & MEM_Int ){ + pCtx->pOut->flags &= ~MEM_Int; + pCtx->pOut->flags |= MEM_IntReal; + } +} +#endif + + /* ** This function is called after a transaction has been committed. It ** invokes callbacks registered with sqlite3_wal_hook() as required.@@ -80990,7 +82110,7 @@ p->rc = SQLITE_NOMEM;
return SQLITE_NOMEM_BKPT; } - if( p->pc<=0 && p->expired ){ + if( p->pc<0 && p->expired ){ p->rc = SQLITE_SCHEMA; rc = SQLITE_ERROR; goto end_of_step;@@ -81009,7 +82129,7 @@ || (db->nDeferredCons==0 && db->nDeferredImmCons==0)
); #ifndef SQLITE_OMIT_TRACE - if( (db->xProfile || (db->mTrace & SQLITE_TRACE_PROFILE)!=0) + if( (db->mTrace & (SQLITE_TRACE_PROFILE|SQLITE_TRACE_XPROFILE))!=0 && !db->init.busy && p->zSql ){ sqlite3OsCurrentTimeInt64(db->pVfs, &p->startTime); }else{@@ -81036,16 +82156,18 @@ rc = sqlite3VdbeExec(p);
db->nVdbeExec--; } + if( rc!=SQLITE_ROW ){ #ifndef SQLITE_OMIT_TRACE - /* If the statement completed successfully, invoke the profile callback */ - if( rc!=SQLITE_ROW ) checkProfileCallback(db, p); + /* If the statement completed successfully, invoke the profile callback */ + checkProfileCallback(db, p); #endif - if( rc==SQLITE_DONE && db->autoCommit ){ - assert( p->rc==SQLITE_OK ); - p->rc = doWalCallbacks(db); - if( p->rc!=SQLITE_OK ){ - rc = SQLITE_ERROR; + if( rc==SQLITE_DONE && db->autoCommit ){ + assert( p->rc==SQLITE_OK ); + p->rc = doWalCallbacks(db); + if( p->rc!=SQLITE_OK ){ + sqlite3Dequote(zType); + } } }@@ -81065,9 +82187,9 @@ assert( rc==SQLITE_ROW || rc==SQLITE_DONE || rc==SQLITE_ERROR
|| (rc&0xff)==SQLITE_BUSY || rc==SQLITE_MISUSE ); assert( (p->rc!=SQLITE_ROW && p->rc!=SQLITE_DONE) || p->rc==p->rcApp ); - if( (p->prepFlags & SQLITE_PREPARE_SAVESQL)!=0 - && rc!=SQLITE_ROW - && rc!=SQLITE_DONE + if( rc!=SQLITE_ROW + && rc!=SQLITE_DONE + && (p->prepFlags & SQLITE_PREPARE_SAVESQL)!=0 ){ /* If this statement was prepared using saved SQL and an ** error has occurred, then return the error code in p->rc to the@@ -81523,10 +82645,10 @@ ** If the result is not a simple column reference (if it is an expression
** or a constant) then useTypes 2, 3, and 4 return NULL. */ static const void *columnName( - sqlite3_stmt *pStmt, - int N, - const void *(*xFunc)(Mem*), - int useType + sqlite3_stmt *pStmt, /* The statement */ + int N, /* Which column to get the name for */ + int useUtf16, /* True to return the name as UTF16 */ + int useType /* What type of name */ ){ const void *ret; Vdbe *p;@@ -81547,8 +82669,15 @@ if( N<n && N>=0 ){
N += useType*n; sqlite3_mutex_enter(db->mutex); assert( db->mallocFailed==0 ); - ret = xFunc(&p->aColName[N]); - /* A malloc may have failed inside of the xFunc() call. If this +#ifndef SQLITE_OMIT_UTF16 + if( useUtf16 ){ + ret = sqlite3_value_text16((sqlite3_value*)&p->aColName[N]); + }else +#endif + { + ret = sqlite3_value_text((sqlite3_value*)&p->aColName[N]); + } + /* A malloc may have failed inside of the _text() call. If this ** is the case, clear the mallocFailed flag and return NULL. */ if( db->mallocFailed ){@@ -81565,13 +82694,11 @@ ** Return the name of the Nth column of the result set returned by SQL
** statement pStmt. */ SQLITE_API const char *sqlite3_column_name(sqlite3_stmt *pStmt, int N){ - return columnName( - pStmt, N, (const void*(*)(Mem*))sqlite3_value_text, COLNAME_NAME); + return columnName(pStmt, N, 0, COLNAME_NAME); } #ifndef SQLITE_OMIT_UTF16 SQLITE_API const void *sqlite3_column_name16(sqlite3_stmt *pStmt, int N){ - return columnName( - pStmt, N, (const void*(*)(Mem*))sqlite3_value_text16, COLNAME_NAME); + return columnName(pStmt, N, 1, COLNAME_NAME); } #endif@@ -81590,13 +82717,11 @@ ** Return the column declaration type (if applicable) of the 'i'th column
** of the result set of SQL statement pStmt. */ SQLITE_API const char *sqlite3_column_decltype(sqlite3_stmt *pStmt, int N){ - return columnName( - pStmt, N, (const void*(*)(Mem*))sqlite3_value_text, COLNAME_DECLTYPE); + return columnName(pStmt, N, 0, COLNAME_DECLTYPE); } #ifndef SQLITE_OMIT_UTF16 SQLITE_API const void *sqlite3_column_decltype16(sqlite3_stmt *pStmt, int N){ - return columnName( - pStmt, N, (const void*(*)(Mem*))sqlite3_value_text16, COLNAME_DECLTYPE); + return columnName(pStmt, N, 1, COLNAME_DECLTYPE); } #endif /* SQLITE_OMIT_UTF16 */ #endif /* SQLITE_OMIT_DECLTYPE */@@ -81608,13 +82733,11 @@ ** NULL is returned if the result column is an expression or constant or
** anything else which is not an unambiguous reference to a database column. */ SQLITE_API const char *sqlite3_column_database_name(sqlite3_stmt *pStmt, int N){ - return columnName( - pStmt, N, (const void*(*)(Mem*))sqlite3_value_text, COLNAME_DATABASE); + return columnName(pStmt, N, 0, COLNAME_DATABASE); } #ifndef SQLITE_OMIT_UTF16 SQLITE_API const void *sqlite3_column_database_name16(sqlite3_stmt *pStmt, int N){ - return columnName( - pStmt, N, (const void*(*)(Mem*))sqlite3_value_text16, COLNAME_DATABASE); + return columnName(pStmt, N, 1, COLNAME_DATABASE); } #endif /* SQLITE_OMIT_UTF16 */@@ -81624,13 +82747,11 @@ ** NULL is returned if the result column is an expression or constant or
** anything else which is not an unambiguous reference to a database column. */ SQLITE_API const char *sqlite3_column_table_name(sqlite3_stmt *pStmt, int N){ - return columnName( - pStmt, N, (const void*(*)(Mem*))sqlite3_value_text, COLNAME_TABLE); + return columnName(pStmt, N, 0, COLNAME_TABLE); } #ifndef SQLITE_OMIT_UTF16 SQLITE_API const void *sqlite3_column_table_name16(sqlite3_stmt *pStmt, int N){ - return columnName( - pStmt, N, (const void*(*)(Mem*))sqlite3_value_text16, COLNAME_TABLE); + return columnName(pStmt, N, 1, COLNAME_TABLE); } #endif /* SQLITE_OMIT_UTF16 */@@ -81640,13 +82761,11 @@ ** NULL is returned if the result column is an expression or constant or
** anything else which is not an unambiguous reference to a database column. */ SQLITE_API const char *sqlite3_column_origin_name(sqlite3_stmt *pStmt, int N){ - return columnName( - pStmt, N, (const void*(*)(Mem*))sqlite3_value_text, COLNAME_COLUMN); + return columnName(pStmt, N, 0, COLNAME_COLUMN); } #ifndef SQLITE_OMIT_UTF16 SQLITE_API const void *sqlite3_column_origin_name16(sqlite3_stmt *pStmt, int N){ - return columnName( - pStmt, N, (const void*(*)(Mem*))sqlite3_value_text16, COLNAME_COLUMN); + return columnName(pStmt, N, 1, COLNAME_COLUMN); } #endif /* SQLITE_OMIT_UTF16 */ #endif /* SQLITE_ENABLE_COLUMN_METADATA */@@ -81689,7 +82808,7 @@ i--;
pVar = &p->aVar[i]; sqlite3VdbeMemRelease(pVar); pVar->flags = MEM_Null; - sqlite3Error(p->db, SQLITE_OK); + p->db->errCode = SQLITE_OK; /* If the bit corresponding to this variable in Vdbe.expmask is set, then ** binding a new value to this variable invalidates the current query plan.@@ -82015,6 +83134,14 @@ return pStmt ? ((Vdbe*)pStmt)->readOnly : 1;
} /* +** Return 1 if the statement is an EXPLAIN and return 2 if the +** statement is an EXPLAIN QUERY PLAN +*/ +SQLITE_API int sqlite3_stmt_isexplain(sqlite3_stmt *pStmt){ + return pStmt ? ((Vdbe*)pStmt)->explain : 0; +} + +/* ** Return true if the prepared statement is in need of being reset. */ SQLITE_API int sqlite3_stmt_busy(sqlite3_stmt *pStmt){@@ -82108,6 +83235,22 @@ }
return z; #endif } + +#ifdef SQLITE_ENABLE_NORMALIZE +/* +** Return the normalized SQL associated with a prepared statement. +*/ +SQLITE_API const char *sqlite3_normalized_sql(sqlite3_stmt *pStmt){ + Vdbe *p = (Vdbe *)pStmt; + if( p==0 ) return 0; + if( p->zNormSql==0 && ALWAYS(p->zSql!=0) ){ + sqlite3_mutex_enter(p->db->mutex); + p->zNormSql = sqlite3Normalize(p, p->zSql); + sqlite3_mutex_leave(p->db->mutex); + } + return p->zNormSql; +} +#endif /* SQLITE_ENABLE_NORMALIZE */ #ifdef SQLITE_ENABLE_PREUPDATE_HOOK /*@@ -82179,7 +83322,9 @@ sqlite3VdbeMemSetInt64(pMem, p->iKey1);
}else if( iIdx>=p->pUnpacked->nField ){ *ppValue = (sqlite3_value *)columnNullValue(); }else if( p->pTab->aCol[iIdx].affinity==SQLITE_AFF_REAL ){ - if( pMem->flags & MEM_Int ){ + if( pMem->flags & (MEM_Int|MEM_IntReal) ){ + testcase( pMem->flags & MEM_Int ); + testcase( pMem->flags & MEM_IntReal ); sqlite3VdbeMemRealify(pMem); } }@@ -82498,7 +83643,7 @@ assert( idx>0 && idx<=p->nVar );
pVar = &p->aVar[idx-1]; if( pVar->flags & MEM_Null ){ sqlite3_str_append(&out, "NULL", 4); - }else if( pVar->flags & MEM_Int ){ + }else if( pVar->flags & (MEM_Int|MEM_IntReal) ){ sqlite3_str_appendf(&out, "%lld", pVar->u.i); }else if( pVar->flags & MEM_Real ){ sqlite3_str_appendf(&out, "%!.15g", pVar->u.r);@@ -82687,12 +83832,20 @@ ** Invoke the VDBE coverage callback, if that callback is defined. This
** feature is used for test suite validation only and does not appear an ** production builds. ** -** M is an integer between 2 and 4. 2 indicates a ordinary two-way -** branch (I=0 means fall through and I=1 means taken). 3 indicates -** a 3-way branch where the third way is when one of the operands is -** NULL. 4 indicates the OP_Jump instruction which has three destinations -** depending on whether the first operand is less than, equal to, or greater -** than the second. +** M is the type of branch. I is the direction taken for this instance of +** the branch. +** +** M: 2 - two-way branch (I=0: fall-thru 1: jump ) +** 3 - two-way + NULL (I=0: fall-thru 1: jump 2: NULL ) +** 4 - OP_Jump (I=0: jump p1 1: jump p2 2: jump p3) +** +** In other words, if M is 2, then I is either 0 (for fall-through) or +** 1 (for when the branch is taken). If M is 3, the I is 0 for an +** ordinary fall-through, I is 1 if the branch was taken, and I is 2 +** if the result of comparison is NULL. For M=3, I=2 the jump may or +** may not be taken, depending on the SQLITE_JUMPIFNULL flags in p5. +** When M is 4, that means that an OP_Jump is being run. I is 0, 1, or 2 +** depending on if the operands are less than, equal, or greater than. ** ** iSrcLine is the source code line (from the __LINE__ macro) that ** generated the VDBE instruction combined with flag bits. The source@@ -82703,9 +83856,9 @@ ** always taken, the flags should be 0x05 since the fall-through and
** alternate branch are never taken. If a branch is never taken then ** flags should be 0x06 since only the fall-through approach is allowed. ** -** Bit 0x04 of the flags indicates an OP_Jump opcode that is only +** Bit 0x08 of the flags indicates an OP_Jump opcode that is only ** interested in equal or not-equal. In other words, I==0 and I==2 -** should be treated the same. +** should be treated as equivalent ** ** Since only a line number is retained, not the filename, this macro ** only works for amalgamation builds. But that is ok, since these macros@@ -82729,6 +83882,18 @@ ** away. */
mNever = iSrcLine >> 24; assert( (I & mNever)==0 ); if( sqlite3GlobalConfig.xVdbeBranch==0 ) return; /*NO_TEST*/ + /* Invoke the branch coverage callback with three arguments: + ** iSrcLine - the line number of the VdbeCoverage() macro, with + ** flags removed. + ** I - Mask of bits 0x07 indicating which cases are are + ** fulfilled by this instance of the jump. 0x01 means + ** fall-thru, 0x02 means taken, 0x04 means NULL. Any + ** impossible cases (ex: if the comparison is never NULL) + ** are filled in automatically so that the coverage + ** measurement logic does not flag those impossible cases + ** as missed coverage. + ** M - Type of jump. Same as M argument above + */ I |= mNever; if( M==2 ) I |= 0x04; if( M==4 ){@@ -82739,14 +83904,6 @@ sqlite3GlobalConfig.xVdbeBranch(sqlite3GlobalConfig.pVdbeBranchArg,
iSrcLine&0xffffff, I, M); } #endif - -/* -** Convert the given register into a string if it isn't one -** already. Return non-zero if a malloc() fails. -*/ -#define Stringify(P, enc) \ - if(((P)->flags&(MEM_Str|MEM_Blob))==0 && sqlite3VdbeMemStringify(P,enc,0)) \ - { goto no_mem; } /* ** An ephemeral string value (signified by the MEM_Ephem flag) contains@@ -82805,6 +83962,11 @@ (eCurType==CURTYPE_BTREE?sqlite3BtreeCursorSize():0);
assert( iCur>=0 && iCur<p->nCursor ); if( p->apCsr[iCur] ){ /*OPTIMIZATION-IF-FALSE*/ + /* Before calling sqlite3VdbeFreeCursor(), ensure the isEphemeral flag + ** is clear. Otherwise, if this is an ephemeral cursor created by + ** OP_OpenDup, the cursor will not be closed and will still be part + ** of a BtShared.pCursor list. */ + if( p->apCsr[iCur]->pBtx==0 ) p->apCsr[iCur]->isEphemeral = 0; sqlite3VdbeFreeCursor(p, p->apCsr[iCur]); p->apCsr[iCur] = 0; }@@ -82825,6 +83987,21 @@ return pCx;
} /* +** The string in pRec is known to look like an integer and to have a +** floating point value of rValue. Return true and set *piValue to the +** integer value if the string is in range to be an integer. Otherwise, +** return false. +*/ +static int alsoAnInt(Mem *pRec, double rValue, i64 *piValue){ + i64 iValue = (double)rValue; + if( sqlite3RealSameAsInt(rValue,iValue) ){ + *piValue = iValue; + return 1; + } + return 0==sqlite3Atoi64(pRec->z, piValue, pRec->n, pRec->enc); +} + +/* ** Try to convert a value into a numeric representation if we can ** do so without loss of information. In other words, if the string ** looks like a number, convert it into a number. If it does not@@ -82841,12 +84018,12 @@ ** if there is an exact integer representation of the quantity.
*/ static void applyNumericAffinity(Mem *pRec, int bTryForInt){ double rValue; - i64 iValue; u8 enc = pRec->enc; - assert( (pRec->flags & (MEM_Str|MEM_Int|MEM_Real))==MEM_Str ); - if( sqlite3AtoF(pRec->z, &rValue, pRec->n, enc)==0 ) return; - if( 0==sqlite3Atoi64(pRec->z, &iValue, pRec->n, enc) ){ - pRec->u.i = iValue; + int rc; + assert( (pRec->flags & (MEM_Str|MEM_Int|MEM_Real|MEM_IntReal))==MEM_Str ); + rc = sqlite3AtoF(pRec->z, &rValue, pRec->n, enc); + if( rc<=0 ) return; + if( rc==1 && alsoAnInt(pRec, rValue, &pRec->u.i) ){ pRec->flags |= MEM_Int; }else{ pRec->u.r = rValue;@@ -82900,11 +84077,14 @@ ** representation. It would be harmless to repeat the conversion if
** there is already a string rep, but it is pointless to waste those ** CPU cycles. */ if( 0==(pRec->flags&MEM_Str) ){ /*OPTIMIZATION-IF-FALSE*/ - if( (pRec->flags&(MEM_Real|MEM_Int)) ){ + if( (pRec->flags&(MEM_Real|MEM_Int|MEM_IntReal)) ){ + testcase( pRec->flags & MEM_Int ); + testcase( pRec->flags & MEM_Real ); + testcase( pRec->flags & MEM_IntReal ); sqlite3VdbeMemStringify(pRec, enc, 1); } } - pRec->flags &= ~(MEM_Real|MEM_Int); + pRec->flags &= ~(MEM_Real|MEM_Int|MEM_IntReal); } }@@ -82943,12 +84123,21 @@ ** numeric type, if has one. Set the pMem->u.r and pMem->u.i fields
** accordingly. */ static u16 SQLITE_NOINLINE computeNumericType(Mem *pMem){ - assert( (pMem->flags & (MEM_Int|MEM_Real))==0 ); + int rc; + sqlite3_int64 ix; + assert( (pMem->flags & (MEM_Int|MEM_Real|MEM_IntReal))==0 ); assert( (pMem->flags & (MEM_Str|MEM_Blob))!=0 ); - if( sqlite3AtoF(pMem->z, &pMem->u.r, pMem->n, pMem->enc)==0 ){ - return 0; - } - if( sqlite3Atoi64(pMem->z, &pMem->u.i, pMem->n, pMem->enc)==0 ){ + ExpandBlob(pMem); + rc = sqlite3AtoF(pMem->z, &pMem->u.r, pMem->n, pMem->enc); + if( rc<=0 ){ + if( rc==0 && sqlite3Atoi64(pMem->z, &ix, pMem->n, pMem->enc)<=1 ){ + pMem->u.i = ix; + return MEM_Int; + }else{ + return MEM_Real; + } + }else if( rc==1 && sqlite3Atoi64(pMem->z, &ix, pMem->n, pMem->enc)==0 ){ + pMem->u.i = ix; return MEM_Int; } return MEM_Real;@@ -82962,10 +84151,15 @@ ** Unlike applyNumericAffinity(), this routine does not modify pMem->flags.
** But it does set pMem->u.r and pMem->u.i appropriately. */ static u16 numericType(Mem *pMem){ - if( pMem->flags & (MEM_Int|MEM_Real) ){ - return pMem->flags & (MEM_Int|MEM_Real); + if( pMem->flags & (MEM_Int|MEM_Real|MEM_IntReal) ){ + testcase( pMem->flags & MEM_Int ); + testcase( pMem->flags & MEM_Real ); + testcase( pMem->flags & MEM_IntReal ); + return pMem->flags & (MEM_Int|MEM_Real|MEM_IntReal); } if( pMem->flags & (MEM_Str|MEM_Blob) ){ + testcase( pMem->flags & MEM_Str ); + testcase( pMem->flags & MEM_Blob ); return computeNumericType(pMem); } return 0;@@ -83061,6 +84255,8 @@ }else if( p->flags & MEM_Null ){
printf(p->flags & MEM_Zero ? " NULL-nochng" : " NULL"); }else if( (p->flags & (MEM_Int|MEM_Str))==(MEM_Int|MEM_Str) ){ printf(" si:%lld", p->u.i); + }else if( (p->flags & (MEM_IntReal))!=0 ){ + printf(" ir:%lld", p->u.i); }else if( p->flags & MEM_Int ){ printf(" i:%lld", p->u.i); #ifndef SQLITE_OMIT_FLOATING_POINT@@ -83270,6 +84466,15 @@ /*** INSERT STACK UNION HERE ***/
assert( p->magic==VDBE_MAGIC_RUN ); /* sqlite3_step() verifies this */ sqlite3VdbeEnter(p); +#ifndef SQLITE_OMIT_PROGRESS_CALLBACK + if( db->xProgress ){ + u32 iPrior = p->aCounter[SQLITE_STMTSTATUS_VM_STEP]; + assert( 0 < db->nProgressOps ); + nProgressLimit = db->nProgressOps - (iPrior % db->nProgressOps); + }else{ + nProgressLimit = 0xffffffff; + } +#endif if( p->rc==SQLITE_NOMEM ){ /* This happens if a malloc() inside a call to sqlite3_column_text() or ** sqlite3_column_text16() failed. */@@ -83283,15 +84488,6 @@ p->pResultSet = 0;
db->busyHandler.nBusy = 0; if( db->u1.isInterrupted ) goto abort_due_to_interrupt; sqlite3VdbeIOTraceSql(p); -#ifndef SQLITE_OMIT_PROGRESS_CALLBACK - if( db->xProgress ){ - u32 iPrior = p->aCounter[SQLITE_STMTSTATUS_VM_STEP]; - assert( 0 < db->nProgressOps ); - nProgressLimit = db->nProgressOps - (iPrior % db->nProgressOps); - }else{ - nProgressLimit = 0xffffffff; - } -#endif #ifdef SQLITE_DEBUG sqlite3BeginBenignMalloc(); if( p->pc==0@@ -83467,10 +84663,11 @@ ** sqlite3VdbeExec() or since last time the progress callback was called).
** If the progress callback returns non-zero, exit the virtual machine with ** a return code SQLITE_ABORT. */ - if( nVmStep>=nProgressLimit && db->xProgress!=0 ){ + while( nVmStep>=nProgressLimit && db->xProgress!=0 ){ assert( db->nProgressOps!=0 ); - nProgressLimit = nVmStep + db->nProgressOps - (nVmStep%db->nProgressOps); + nProgressLimit += db->nProgressOps; if( db->xProgress(db->pProgressArg) ){ + nProgressLimit = 0xffffffff; rc = SQLITE_INTERRUPT; goto abort_due_to_error; }@@ -83749,6 +84946,7 @@ #ifndef SQLITE_OMIT_UTF16
if( encoding!=SQLITE_UTF8 ){ rc = sqlite3VdbeMemSetStr(pOut, pOp->p4.z, -1, SQLITE_UTF8, SQLITE_STATIC); assert( rc==SQLITE_OK || rc==SQLITE_TOOBIG ); + if( rc ) goto too_big; if( SQLITE_OK!=sqlite3VdbeChangeEncoding(pOut, encoding) ) goto no_mem; assert( pOut->szMalloc>0 && pOut->zMalloc==pOut->z ); assert( VdbeMemDynamic(pOut)==0 );@@ -83761,7 +84959,6 @@ pOp->p4type = P4_DYNAMIC;
pOp->p4.z = pOut->z; pOp->p1 = pOut->n; } - testcase( rc==SQLITE_TOOBIG ); #endif if( pOp->p1>db->aLimit[SQLITE_LIMIT_LENGTH] ){ goto too_big;@@ -83883,7 +85080,10 @@ if( sqlite3VdbeMemTooBig(pVar) ){
goto too_big; } pOut = &aMem[pOp->p2]; - sqlite3VdbeMemShallowCopy(pOut, pVar, MEM_Static); + if( VdbeMemDynamic(pOut) ) sqlite3VdbeMemSetNull(pOut); + memcpy(pOut, pVar, MEMCELLSIZE); + pOut->flags &= ~(MEM_Dyn|MEM_Ephem); + pOut->flags |= MEM_Static|MEM_FromBind; UPDATE_MAX_BLOBSIZE(pOut); break; }@@ -84016,18 +85216,6 @@ assert( p->nResColumn==pOp->p2 );
assert( pOp->p1>0 ); assert( pOp->p1+pOp->p2<=(p->nMem+1 - p->nCursor)+1 ); -#ifndef SQLITE_OMIT_PROGRESS_CALLBACK - /* Run the progress counter just before returning. - */ - if( db->xProgress!=0 - && nVmStep>=nProgressLimit - && db->xProgress(db->pProgressArg)!=0 - ){ - rc = SQLITE_INTERRUPT; - goto abort_due_to_error; - } -#endif - /* If this statement has violated immediate foreign key constraints, do ** not return the number of rows modified. And do not RELEASE the statement ** transaction. It needs to be rolled back. */@@ -84099,33 +85287,57 @@ ** if P3 is the same register as P2, the implementation is able
** to avoid a memcpy(). */ case OP_Concat: { /* same as TK_CONCAT, in1, in2, out3 */ - i64 nByte; + i64 nByte; /* Total size of the output string or blob */ + u16 flags1; /* Initial flags for P1 */ + u16 flags2; /* Initial flags for P2 */ pIn1 = &aMem[pOp->p1]; pIn2 = &aMem[pOp->p2]; pOut = &aMem[pOp->p3]; + testcase( pIn1==pIn2 ); + testcase( pOut==pIn2 ); assert( pIn1!=pOut ); - if( (pIn1->flags | pIn2->flags) & MEM_Null ){ + flags1 = pIn1->flags; + testcase( flags1 & MEM_Null ); + testcase( pIn2->flags & MEM_Null ); + if( (flags1 | pIn2->flags) & MEM_Null ){ sqlite3VdbeMemSetNull(pOut); break; } - if( ExpandBlob(pIn1) || ExpandBlob(pIn2) ) goto no_mem; - Stringify(pIn1, encoding); - Stringify(pIn2, encoding); + if( (flags1 & (MEM_Str|MEM_Blob))==0 ){ + if( sqlite3VdbeMemStringify(pIn1,encoding,0) ) goto no_mem; + flags1 = pIn1->flags & ~MEM_Str; + }else if( (flags1 & MEM_Zero)!=0 ){ + if( sqlite3VdbeMemExpandBlob(pIn1) ) goto no_mem; + flags1 = pIn1->flags & ~MEM_Str; + } + flags2 = pIn2->flags; + if( (flags2 & (MEM_Str|MEM_Blob))==0 ){ + if( sqlite3VdbeMemStringify(pIn2,encoding,0) ) goto no_mem; + flags2 = pIn2->flags & ~MEM_Str; + }else if( (flags2 & MEM_Zero)!=0 ){ + if( sqlite3VdbeMemExpandBlob(pIn2) ) goto no_mem; + flags2 = pIn2->flags & ~MEM_Str; + } nByte = pIn1->n + pIn2->n; if( nByte>db->aLimit[SQLITE_LIMIT_LENGTH] ){ goto too_big; } - if( sqlite3VdbeMemGrow(pOut, (int)nByte+2, pOut==pIn2) ){ + if( sqlite3VdbeMemGrow(pOut, (int)nByte+3, pOut==pIn2) ){ goto no_mem; } MemSetTypeFlag(pOut, MEM_Str); if( pOut!=pIn2 ){ memcpy(pOut->z, pIn2->z, pIn2->n); + assert( (pIn2->flags & MEM_Dyn) == (flags2 & MEM_Dyn) ); + pIn2->flags = flags2; } memcpy(&pOut->z[pIn2->n], pIn1->z, pIn1->n); + assert( (pIn1->flags & MEM_Dyn) == (flags1 & MEM_Dyn) ); + pIn1->flags = flags1; pOut->z[nByte]=0; pOut->z[nByte+1] = 0; + pOut->z[nByte+2] = 0; pOut->flags |= MEM_Term; pOut->n = (int)nByte; pOut->enc = encoding;@@ -84176,7 +85388,6 @@ case OP_Subtract: /* same as TK_MINUS, in1, in2, out3 */
case OP_Multiply: /* same as TK_STAR, in1, in2, out3 */ case OP_Divide: /* same as TK_SLASH, in1, in2, out3 */ case OP_Remainder: { /* same as TK_REM, in1, in2, out3 */ - char bIntint; /* Started out as two integer operands */ u16 flags; /* Combined MEM_* flags from both inputs */ u16 type1; /* Numeric type of left operand */ u16 type2; /* Numeric type of right operand */@@ -84194,7 +85405,6 @@ flags = pIn1->flags | pIn2->flags;
if( (type1 & type2 & MEM_Int)!=0 ){ iA = pIn1->u.i; iB = pIn2->u.i; - bIntint = 1; switch( pOp->opcode ){ case OP_Add: if( sqlite3AddInt64(&iB,iA) ) goto fp_math; break; case OP_Subtract: if( sqlite3SubInt64(&iB,iA) ) goto fp_math; break;@@ -84217,7 +85427,6 @@ MemSetTypeFlag(pOut, MEM_Int);
}else if( (flags & MEM_Null)!=0 ){ goto arithmetic_result_is_null; }else{ - bIntint = 0; fp_math: rA = sqlite3VdbeRealValue(pIn1); rB = sqlite3VdbeRealValue(pIn2);@@ -84232,8 +85441,8 @@ rB /= rA;
break; } default: { - iA = (i64)rA; - iB = (i64)rB; + iA = sqlite3VdbeIntValue(pIn1); + iB = sqlite3VdbeIntValue(pIn2); if( iA==0 ) goto arithmetic_result_is_null; if( iA==-1 ) iA = 1; rB = (double)(iB % iA);@@ -84249,9 +85458,6 @@ goto arithmetic_result_is_null;
} pOut->u.r = rB; MemSetTypeFlag(pOut, MEM_Real); - if( ((type1|type2)&MEM_Real)==0 && !bIntint ){ - sqlite3VdbeIntegerAffinity(pOut); - } #endif } break;@@ -84393,8 +85599,8 @@ case OP_MustBeInt: { /* jump, in1 */
pIn1 = &aMem[pOp->p1]; if( (pIn1->flags & MEM_Int)==0 ){ applyAffinity(pIn1, SQLITE_AFF_NUMERIC, encoding); - VdbeBranchTaken((pIn1->flags&MEM_Int)==0, 2); if( (pIn1->flags & MEM_Int)==0 ){ + VdbeBranchTaken(1, 2); if( pOp->p2==0 ){ rc = SQLITE_MISMATCH; goto abort_due_to_error;@@ -84403,6 +85609,7 @@ goto jump_to_p2;
} } } + VdbeBranchTaken(0, 2); MemSetTypeFlag(pIn1, MEM_Int); break; }@@ -84419,7 +85626,9 @@ ** to have only a real value.
*/ case OP_RealAffinity: { /* in1 */ pIn1 = &aMem[pOp->p1]; - if( pIn1->flags & MEM_Int ){ + if( pIn1->flags & (MEM_Int|MEM_IntReal) ){ + testcase( pIn1->flags & MEM_Int ); + testcase( pIn1->flags & MEM_IntReal ); sqlite3VdbeMemRealify(pIn1); } break;@@ -84577,15 +85786,15 @@ /* If SQLITE_NULLEQ is set (which will only happen if the operator is
** OP_Eq or OP_Ne) then take the jump or not depending on whether ** or not both operands are null. */ - assert( pOp->opcode==OP_Eq || pOp->opcode==OP_Ne ); assert( (flags1 & MEM_Cleared)==0 ); - assert( (pOp->p5 & SQLITE_JUMPIFNULL)==0 ); + assert( (pOp->p5 & SQLITE_JUMPIFNULL)==0 || CORRUPT_DB ); + testcase( (pOp->p5 & SQLITE_JUMPIFNULL)!=0 ); if( (flags1&flags3&MEM_Null)!=0 && (flags3&MEM_Cleared)==0 ){ res = 0; /* Operands are equal */ }else{ - res = 1; /* Operands are not equal */ + res = ((flags3 & MEM_Null) ? -1 : +1); /* Operands are not equal */ } }else{ /* SQLITE_NULLEQ is clear and at least one operand is NULL,@@ -84611,7 +85820,7 @@ /* Neither operand is NULL. Do a comparison. */
affinity = pOp->p5 & SQLITE_AFF_MASK; if( affinity>=SQLITE_AFF_NUMERIC ){ if( (flags1 | flags3)&MEM_Str ){ - if( (flags1 & (MEM_Int|MEM_Real|MEM_Str))==MEM_Str ){ + if( (flags1 & (MEM_Int|MEM_IntReal|MEM_Real|MEM_Str))==MEM_Str ){ applyNumericAffinity(pIn1,0); assert( flags3==pIn3->flags ); /* testcase( flags3!=pIn3->flags );@@ -84621,7 +85830,7 @@ ** is essentially a no-op. But, it provides defense-in-depth
** in case our analysis is incorrect, so it is left in. */ flags3 = pIn3->flags; } - if( (flags3 & (MEM_Int|MEM_Real|MEM_Str))==MEM_Str ){ + if( (flags3 & (MEM_Int|MEM_IntReal|MEM_Real|MEM_Str))==MEM_Str ){ applyNumericAffinity(pIn3,0); } }@@ -84634,17 +85843,19 @@ res = 0;
goto compare_op; } }else if( affinity==SQLITE_AFF_TEXT ){ - if( (flags1 & MEM_Str)==0 && (flags1 & (MEM_Int|MEM_Real))!=0 ){ + if( (flags1 & MEM_Str)==0 && (flags1&(MEM_Int|MEM_Real|MEM_IntReal))!=0 ){ testcase( pIn1->flags & MEM_Int ); testcase( pIn1->flags & MEM_Real ); + testcase( pIn1->flags & MEM_IntReal ); sqlite3VdbeMemStringify(pIn1, encoding, 1); testcase( (flags1&MEM_Dyn) != (pIn1->flags&MEM_Dyn) ); flags1 = (pIn1->flags & ~MEM_TypeMask) | (flags1 & MEM_TypeMask); assert( pIn1!=pIn3 ); } - if( (flags3 & MEM_Str)==0 && (flags3 & (MEM_Int|MEM_Real))!=0 ){ + if( (flags3 & MEM_Str)==0 && (flags3&(MEM_Int|MEM_Real|MEM_IntReal))!=0 ){ testcase( pIn3->flags & MEM_Int ); testcase( pIn3->flags & MEM_Real ); + testcase( pIn3->flags & MEM_IntReal ); sqlite3VdbeMemStringify(pIn3, encoding, 1); testcase( (flags3&MEM_Dyn) != (pIn3->flags&MEM_Dyn) ); flags3 = (pIn3->flags & ~MEM_TypeMask) | (flags3 & MEM_TypeMask);@@ -84703,7 +85914,7 @@ MemSetTypeFlag(pOut, MEM_Int);
pOut->u.i = res2; REGISTER_TRACE(pOp->p2, pOut); }else{ - VdbeBranchTaken(res!=0, (pOp->p5 & SQLITE_NULLEQ)?2:3); + VdbeBranchTaken(res2!=0, (pOp->p5 & SQLITE_NULLEQ)?2:3); if( res2 ){ goto jump_to_p2; }@@ -85253,15 +86464,15 @@ zHdr = zData + pC->iHdrOffset;
zEndHdr = zData + aOffset[0]; testcase( zHdr>=zEndHdr ); do{ - if( (t = zHdr[0])<0x80 ){ + if( (pC->aType[i] = t = zHdr[0])<0x80 ){ zHdr++; offset64 += sqlite3VdbeOneByteSerialTypeLen(t); }else{ zHdr += sqlite3GetVarint32(zHdr, &t); + pC->aType[i] = t; offset64 += sqlite3VdbeSerialTypeLen(t); } - pC->aType[i++] = t; - aOffset[i] = (u32)(offset64 & 0xffffffff); + aOffset[++i] = (u32)(offset64 & 0xffffffff); }while( i<=p2 && zHdr<zEndHdr ); /* The record is corrupt if any of the following are true:@@ -85400,12 +86611,21 @@ assert( zAffinity!=0 );
assert( pOp->p2>0 ); assert( zAffinity[pOp->p2]==0 ); pIn1 = &aMem[pOp->p1]; - do{ + while( 1 /*edit-by-break*/ ){ assert( pIn1 <= &p->aMem[(p->nMem+1 - p->nCursor)] ); assert( memIsValid(pIn1) ); - applyAffinity(pIn1, *(zAffinity++), encoding); + applyAffinity(pIn1, zAffinity[0], encoding); + if( zAffinity[0]==SQLITE_AFF_REAL && (pIn1->flags & MEM_Int)!=0 ){ + /* When applying REAL affinity, if the result is still MEM_Int, + ** indicate that REAL is actually desired */ + pIn1->flags |= MEM_IntReal; + pIn1->flags &= ~MEM_Int; + } + REGISTER_TRACE((int)(pIn1-aMem), pIn1); + zAffinity++; + if( zAffinity[0]==0 ) break; pIn1++; - }while( zAffinity[0] ); + } break; }@@ -85426,7 +86646,6 @@ **
** If P4 is NULL then all index fields have the affinity BLOB. */ case OP_MakeRecord: { - u8 *zNewRecord; /* A buffer to hold the data for the new record */ Mem *pRec; /* The new record */ u64 nData; /* Number of bytes of data space */ int nHdr; /* Number of bytes of header space */@@ -85439,9 +86658,9 @@ Mem *pLast; /* Last field of the record */
int nField; /* Number of fields in the record */ char *zAffinity; /* The affinity string for the record */ int file_format; /* File format to use for encoding */ - int i; /* Space used in zNewRecord[] header */ - int j; /* Space used in zNewRecord[] content */ u32 len; /* Length of a field */ + u8 *zHdr; /* Where to write next byte of the header */ + u8 *zPayload; /* Where to write next byte of the payload */ /* Assuming the record contains N fields, the record format looks ** like this:@@ -85480,7 +86699,14 @@ assert( pData0<=pLast );
if( zAffinity ){ pRec = pData0; do{ - applyAffinity(pRec++, *(zAffinity++), encoding); + applyAffinity(pRec, zAffinity[0], encoding); + if( zAffinity[0]==SQLITE_AFF_REAL && (pRec->flags & MEM_Int) ){ + pRec->flags |= MEM_IntReal; + pRec->flags &= ~(MEM_Int); + } + REGISTER_TRACE((int)(pRec-aMem), pRec); + zAffinity++; + pRec++; assert( zAffinity[0]==0 || pRec<=pLast ); }while( zAffinity[0] ); }@@ -85548,46 +86774,54 @@ nHdr += nVarint;
if( nVarint<sqlite3VarintLen(nHdr) ) nHdr++; } nByte = nHdr+nData; - if( nByte+nZero>db->aLimit[SQLITE_LIMIT_LENGTH] ){ - goto too_big; - } /* Make sure the output register has a buffer large enough to store ** the new record. The output register (pOp->p3) is not allowed to ** be one of the input registers (because the following call to ** sqlite3VdbeMemClearAndResize() could clobber the value before it is used). */ - if( sqlite3VdbeMemClearAndResize(pOut, (int)nByte) ){ - goto no_mem; + if( nByte+nZero<=pOut->szMalloc ){ + /* The output register is already large enough to hold the record. + ** No error checks or buffer enlargement is required */ + pOut->z = pOut->zMalloc; + }else{ + /* Need to make sure that the output is not too big and then enlarge + ** the output register to hold the full result */ + if( nByte+nZero>db->aLimit[SQLITE_LIMIT_LENGTH] ){ + goto too_big; + } + if( sqlite3VdbeMemClearAndResize(pOut, (int)nByte) ){ + goto no_mem; + } } - zNewRecord = (u8 *)pOut->z; + pOut->n = (int)nByte; + pOut->flags = MEM_Blob; + if( nZero ){ + pOut->u.nZero = nZero; + pOut->flags |= MEM_Zero; + } + UPDATE_MAX_BLOBSIZE(pOut); + zHdr = (u8 *)pOut->z; + zPayload = zHdr + nHdr; /* Write the record */ - i = putVarint32(zNewRecord, nHdr); - j = nHdr; + zHdr += putVarint32(zHdr, nHdr); assert( pData0<=pLast ); pRec = pData0; do{ serial_type = pRec->uTemp; /* EVIDENCE-OF: R-06529-47362 Following the size varint are one or more ** additional varints, one per column. */ - i += putVarint32(&zNewRecord[i], serial_type); /* serial type */ + zHdr += putVarint32(zHdr, serial_type); /* serial type */ /* EVIDENCE-OF: R-64536-51728 The values for each column in the record ** immediately follow the header. */ - j += sqlite3VdbeSerialPut(&zNewRecord[j], pRec, serial_type); /* content */ + zPayload += sqlite3VdbeSerialPut(zPayload, pRec, serial_type); /* content */ }while( (++pRec)<=pLast ); - assert( i==nHdr ); - assert( j==nByte ); + assert( nHdr==(int)(zHdr - (u8*)pOut->z) ); + assert( nByte==(int)(zPayload - (u8*)pOut->z) ); assert( pOp->p3>0 && pOp->p3<=(p->nMem+1 - p->nCursor) ); - pOut->n = (int)nByte; - pOut->flags = MEM_Blob; - if( nZero ){ - pOut->u.nZero = nZero; - pOut->flags |= MEM_Zero; - } REGISTER_TRACE(pOp->p3, pOut); - UPDATE_MAX_BLOBSIZE(pOut); break; }@@ -85617,8 +86851,9 @@
/* Opcode: Savepoint P1 * * P4 * ** ** Open, release or rollback the savepoint named by parameter P4, depending -** on the value of P1. To open a new savepoint, P1==0. To release (commit) an -** existing savepoint, P1==1, or to rollback an existing savepoint P1==2. +** on the value of P1. To open a new savepoint set P1==0 (SAVEPOINT_BEGIN). +** To release (commit) an existing savepoint set P1==1 (SAVEPOINT_RELEASE). +** To rollback an existing savepoint set P1==2 (SAVEPOINT_ROLLBACK). */ case OP_Savepoint: { int p1; /* Value of P1 operand */@@ -85686,6 +86921,7 @@ pNew->nDeferredImmCons = db->nDeferredImmCons;
} } }else{ + assert( p1==SAVEPOINT_RELEASE || p1==SAVEPOINT_ROLLBACK ); iSavepoint = 0; /* Find the named savepoint. If there is no such savepoint, then an@@ -85739,6 +86975,7 @@ isSchemaChange==0);
if( rc!=SQLITE_OK ) goto abort_due_to_error; } }else{ + assert( p1==SAVEPOINT_RELEASE ); isSchemaChange = 0; } for(ii=0; ii<db->nDb; ii++){@@ -85775,6 +87012,7 @@ if( !isTransaction ){
db->nSavepoint--; } }else{ + assert( p1==SAVEPOINT_ROLLBACK ); db->nDeferredCons = pSavepoint->nDeferredCons; db->nDeferredImmCons = pSavepoint->nDeferredImmCons; }@@ -86255,7 +87493,9 @@ pCx->nullRow = 1;
pCx->isEphemeral = 1; pCx->pKeyInfo = pOrig->pKeyInfo; pCx->isTable = pOrig->isTable; - rc = sqlite3BtreeCursor(pOrig->pBtx, MASTER_ROOT, BTREE_WRCSR, + pCx->pgnoRoot = pOrig->pgnoRoot; + pCx->isOrdered = pOrig->isOrdered; + rc = sqlite3BtreeCursor(pOrig->pBtx, pCx->pgnoRoot, BTREE_WRCSR, pCx->pKeyInfo, pCx->uc.pCursor); /* The sqlite3BtreeCursor() routine can only fail for the first cursor ** opened for a database. Since there is already an open cursor when this@@ -86272,6 +87512,9 @@ ** Open a new cursor P1 to a transient table.
** The cursor is always opened read/write ev